{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# 模型评估方法",
   "id": "f47efd1a92227c44"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:41:29.651324Z",
     "start_time": "2025-03-12T08:41:29.150327Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "# 设置图片的相关字体大小\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "import warnings     # 使程序运行时可以捕捉并处理一些非致命性的问题，而不中断程序的执行\n",
    "warnings.filterwarnings('ignore')       # 忽略所有的警告信息，避免在执行代码时出现不必要地输出或干扰\n",
    "np.random.seed(42)      # 可以确保每次运行代码时生成的随机数序列是相同的"
   ],
   "id": "b3f3be727788465a",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "## 数据集读取\n",
    "- Mnist数据是图像数据：(28, 28, 1)的灰度图"
   ],
   "id": "9dd82fdddb6eea70"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:41:37.296713Z",
     "start_time": "2025-03-12T08:41:32.484908Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "mnist = fetch_openml('mnist_784')\n",
    "# 共七万个数字，每个数字有784给像素点\n",
    "mnist"
   ],
   "id": "8a4643548f6e922",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'data':        pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  pixel8  pixel9  \\\n",
       " 0           0       0       0       0       0       0       0       0       0   \n",
       " 1           0       0       0       0       0       0       0       0       0   \n",
       " 2           0       0       0       0       0       0       0       0       0   \n",
       " 3           0       0       0       0       0       0       0       0       0   \n",
       " 4           0       0       0       0       0       0       0       0       0   \n",
       " ...       ...     ...     ...     ...     ...     ...     ...     ...     ...   \n",
       " 69995       0       0       0       0       0       0       0       0       0   \n",
       " 69996       0       0       0       0       0       0       0       0       0   \n",
       " 69997       0       0       0       0       0       0       0       0       0   \n",
       " 69998       0       0       0       0       0       0       0       0       0   \n",
       " 69999       0       0       0       0       0       0       0       0       0   \n",
       " \n",
       "        pixel10  ...  pixel775  pixel776  pixel777  pixel778  pixel779  \\\n",
       " 0            0  ...         0         0         0         0         0   \n",
       " 1            0  ...         0         0         0         0         0   \n",
       " 2            0  ...         0         0         0         0         0   \n",
       " 3            0  ...         0         0         0         0         0   \n",
       " 4            0  ...         0         0         0         0         0   \n",
       " ...        ...  ...       ...       ...       ...       ...       ...   \n",
       " 69995        0  ...         0         0         0         0         0   \n",
       " 69996        0  ...         0         0         0         0         0   \n",
       " 69997        0  ...         0         0         0         0         0   \n",
       " 69998        0  ...         0         0         0         0         0   \n",
       " 69999        0  ...         0         0         0         0         0   \n",
       " \n",
       "        pixel780  pixel781  pixel782  pixel783  pixel784  \n",
       " 0             0         0         0         0         0  \n",
       " 1             0         0         0         0         0  \n",
       " 2             0         0         0         0         0  \n",
       " 3             0         0         0         0         0  \n",
       " 4             0         0         0         0         0  \n",
       " ...         ...       ...       ...       ...       ...  \n",
       " 69995         0         0         0         0         0  \n",
       " 69996         0         0         0         0         0  \n",
       " 69997         0         0         0         0         0  \n",
       " 69998         0         0         0         0         0  \n",
       " 69999         0         0         0         0         0  \n",
       " \n",
       " [70000 rows x 784 columns],\n",
       " 'target': 0        5\n",
       " 1        0\n",
       " 2        4\n",
       " 3        1\n",
       " 4        9\n",
       "         ..\n",
       " 69995    2\n",
       " 69996    3\n",
       " 69997    4\n",
       " 69998    5\n",
       " 69999    6\n",
       " Name: class, Length: 70000, dtype: category\n",
       " Categories (10, object): ['0', '1', '2', '3', ..., '6', '7', '8', '9'],\n",
       " 'frame':        pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  pixel8  pixel9  \\\n",
       " 0           0       0       0       0       0       0       0       0       0   \n",
       " 1           0       0       0       0       0       0       0       0       0   \n",
       " 2           0       0       0       0       0       0       0       0       0   \n",
       " 3           0       0       0       0       0       0       0       0       0   \n",
       " 4           0       0       0       0       0       0       0       0       0   \n",
       " ...       ...     ...     ...     ...     ...     ...     ...     ...     ...   \n",
       " 69995       0       0       0       0       0       0       0       0       0   \n",
       " 69996       0       0       0       0       0       0       0       0       0   \n",
       " 69997       0       0       0       0       0       0       0       0       0   \n",
       " 69998       0       0       0       0       0       0       0       0       0   \n",
       " 69999       0       0       0       0       0       0       0       0       0   \n",
       " \n",
       "        pixel10  ...  pixel776  pixel777  pixel778  pixel779  pixel780  \\\n",
       " 0            0  ...         0         0         0         0         0   \n",
       " 1            0  ...         0         0         0         0         0   \n",
       " 2            0  ...         0         0         0         0         0   \n",
       " 3            0  ...         0         0         0         0         0   \n",
       " 4            0  ...         0         0         0         0         0   \n",
       " ...        ...  ...       ...       ...       ...       ...       ...   \n",
       " 69995        0  ...         0         0         0         0         0   \n",
       " 69996        0  ...         0         0         0         0         0   \n",
       " 69997        0  ...         0         0         0         0         0   \n",
       " 69998        0  ...         0         0         0         0         0   \n",
       " 69999        0  ...         0         0         0         0         0   \n",
       " \n",
       "        pixel781  pixel782  pixel783  pixel784  class  \n",
       " 0             0         0         0         0      5  \n",
       " 1             0         0         0         0      0  \n",
       " 2             0         0         0         0      4  \n",
       " 3             0         0         0         0      1  \n",
       " 4             0         0         0         0      9  \n",
       " ...         ...       ...       ...       ...    ...  \n",
       " 69995         0         0         0         0      2  \n",
       " 69996         0         0         0         0      3  \n",
       " 69997         0         0         0         0      4  \n",
       " 69998         0         0         0         0      5  \n",
       " 69999         0         0         0         0      6  \n",
       " \n",
       " [70000 rows x 785 columns],\n",
       " 'categories': None,\n",
       " 'feature_names': ['pixel1',\n",
       "  'pixel2',\n",
       "  'pixel3',\n",
       "  'pixel4',\n",
       "  'pixel5',\n",
       "  'pixel6',\n",
       "  'pixel7',\n",
       "  'pixel8',\n",
       "  'pixel9',\n",
       "  'pixel10',\n",
       "  'pixel11',\n",
       "  'pixel12',\n",
       "  'pixel13',\n",
       "  'pixel14',\n",
       "  'pixel15',\n",
       "  'pixel16',\n",
       "  'pixel17',\n",
       "  'pixel18',\n",
       "  'pixel19',\n",
       "  'pixel20',\n",
       "  'pixel21',\n",
       "  'pixel22',\n",
       "  'pixel23',\n",
       "  'pixel24',\n",
       "  'pixel25',\n",
       "  'pixel26',\n",
       "  'pixel27',\n",
       "  'pixel28',\n",
       "  'pixel29',\n",
       "  'pixel30',\n",
       "  'pixel31',\n",
       "  'pixel32',\n",
       "  'pixel33',\n",
       "  'pixel34',\n",
       "  'pixel35',\n",
       "  'pixel36',\n",
       "  'pixel37',\n",
       "  'pixel38',\n",
       "  'pixel39',\n",
       "  'pixel40',\n",
       "  'pixel41',\n",
       "  'pixel42',\n",
       "  'pixel43',\n",
       "  'pixel44',\n",
       "  'pixel45',\n",
       "  'pixel46',\n",
       "  'pixel47',\n",
       "  'pixel48',\n",
       "  'pixel49',\n",
       "  'pixel50',\n",
       "  'pixel51',\n",
       "  'pixel52',\n",
       "  'pixel53',\n",
       "  'pixel54',\n",
       "  'pixel55',\n",
       "  'pixel56',\n",
       "  'pixel57',\n",
       "  'pixel58',\n",
       "  'pixel59',\n",
       "  'pixel60',\n",
       "  'pixel61',\n",
       "  'pixel62',\n",
       "  'pixel63',\n",
       "  'pixel64',\n",
       "  'pixel65',\n",
       "  'pixel66',\n",
       "  'pixel67',\n",
       "  'pixel68',\n",
       "  'pixel69',\n",
       "  'pixel70',\n",
       "  'pixel71',\n",
       "  'pixel72',\n",
       "  'pixel73',\n",
       "  'pixel74',\n",
       "  'pixel75',\n",
       "  'pixel76',\n",
       "  'pixel77',\n",
       "  'pixel78',\n",
       "  'pixel79',\n",
       "  'pixel80',\n",
       "  'pixel81',\n",
       "  'pixel82',\n",
       "  'pixel83',\n",
       "  'pixel84',\n",
       "  'pixel85',\n",
       "  'pixel86',\n",
       "  'pixel87',\n",
       "  'pixel88',\n",
       "  'pixel89',\n",
       "  'pixel90',\n",
       "  'pixel91',\n",
       "  'pixel92',\n",
       "  'pixel93',\n",
       "  'pixel94',\n",
       "  'pixel95',\n",
       "  'pixel96',\n",
       "  'pixel97',\n",
       "  'pixel98',\n",
       "  'pixel99',\n",
       "  'pixel100',\n",
       "  'pixel101',\n",
       "  'pixel102',\n",
       "  'pixel103',\n",
       "  'pixel104',\n",
       "  'pixel105',\n",
       "  'pixel106',\n",
       "  'pixel107',\n",
       "  'pixel108',\n",
       "  'pixel109',\n",
       "  'pixel110',\n",
       "  'pixel111',\n",
       "  'pixel112',\n",
       "  'pixel113',\n",
       "  'pixel114',\n",
       "  'pixel115',\n",
       "  'pixel116',\n",
       "  'pixel117',\n",
       "  'pixel118',\n",
       "  'pixel119',\n",
       "  'pixel120',\n",
       "  'pixel121',\n",
       "  'pixel122',\n",
       "  'pixel123',\n",
       "  'pixel124',\n",
       "  'pixel125',\n",
       "  'pixel126',\n",
       "  'pixel127',\n",
       "  'pixel128',\n",
       "  'pixel129',\n",
       "  'pixel130',\n",
       "  'pixel131',\n",
       "  'pixel132',\n",
       "  'pixel133',\n",
       "  'pixel134',\n",
       "  'pixel135',\n",
       "  'pixel136',\n",
       "  'pixel137',\n",
       "  'pixel138',\n",
       "  'pixel139',\n",
       "  'pixel140',\n",
       "  'pixel141',\n",
       "  'pixel142',\n",
       "  'pixel143',\n",
       "  'pixel144',\n",
       "  'pixel145',\n",
       "  'pixel146',\n",
       "  'pixel147',\n",
       "  'pixel148',\n",
       "  'pixel149',\n",
       "  'pixel150',\n",
       "  'pixel151',\n",
       "  'pixel152',\n",
       "  'pixel153',\n",
       "  'pixel154',\n",
       "  'pixel155',\n",
       "  'pixel156',\n",
       "  'pixel157',\n",
       "  'pixel158',\n",
       "  'pixel159',\n",
       "  'pixel160',\n",
       "  'pixel161',\n",
       "  'pixel162',\n",
       "  'pixel163',\n",
       "  'pixel164',\n",
       "  'pixel165',\n",
       "  'pixel166',\n",
       "  'pixel167',\n",
       "  'pixel168',\n",
       "  'pixel169',\n",
       "  'pixel170',\n",
       "  'pixel171',\n",
       "  'pixel172',\n",
       "  'pixel173',\n",
       "  'pixel174',\n",
       "  'pixel175',\n",
       "  'pixel176',\n",
       "  'pixel177',\n",
       "  'pixel178',\n",
       "  'pixel179',\n",
       "  'pixel180',\n",
       "  'pixel181',\n",
       "  'pixel182',\n",
       "  'pixel183',\n",
       "  'pixel184',\n",
       "  'pixel185',\n",
       "  'pixel186',\n",
       "  'pixel187',\n",
       "  'pixel188',\n",
       "  'pixel189',\n",
       "  'pixel190',\n",
       "  'pixel191',\n",
       "  'pixel192',\n",
       "  'pixel193',\n",
       "  'pixel194',\n",
       "  'pixel195',\n",
       "  'pixel196',\n",
       "  'pixel197',\n",
       "  'pixel198',\n",
       "  'pixel199',\n",
       "  'pixel200',\n",
       "  'pixel201',\n",
       "  'pixel202',\n",
       "  'pixel203',\n",
       "  'pixel204',\n",
       "  'pixel205',\n",
       "  'pixel206',\n",
       "  'pixel207',\n",
       "  'pixel208',\n",
       "  'pixel209',\n",
       "  'pixel210',\n",
       "  'pixel211',\n",
       "  'pixel212',\n",
       "  'pixel213',\n",
       "  'pixel214',\n",
       "  'pixel215',\n",
       "  'pixel216',\n",
       "  'pixel217',\n",
       "  'pixel218',\n",
       "  'pixel219',\n",
       "  'pixel220',\n",
       "  'pixel221',\n",
       "  'pixel222',\n",
       "  'pixel223',\n",
       "  'pixel224',\n",
       "  'pixel225',\n",
       "  'pixel226',\n",
       "  'pixel227',\n",
       "  'pixel228',\n",
       "  'pixel229',\n",
       "  'pixel230',\n",
       "  'pixel231',\n",
       "  'pixel232',\n",
       "  'pixel233',\n",
       "  'pixel234',\n",
       "  'pixel235',\n",
       "  'pixel236',\n",
       "  'pixel237',\n",
       "  'pixel238',\n",
       "  'pixel239',\n",
       "  'pixel240',\n",
       "  'pixel241',\n",
       "  'pixel242',\n",
       "  'pixel243',\n",
       "  'pixel244',\n",
       "  'pixel245',\n",
       "  'pixel246',\n",
       "  'pixel247',\n",
       "  'pixel248',\n",
       "  'pixel249',\n",
       "  'pixel250',\n",
       "  'pixel251',\n",
       "  'pixel252',\n",
       "  'pixel253',\n",
       "  'pixel254',\n",
       "  'pixel255',\n",
       "  'pixel256',\n",
       "  'pixel257',\n",
       "  'pixel258',\n",
       "  'pixel259',\n",
       "  'pixel260',\n",
       "  'pixel261',\n",
       "  'pixel262',\n",
       "  'pixel263',\n",
       "  'pixel264',\n",
       "  'pixel265',\n",
       "  'pixel266',\n",
       "  'pixel267',\n",
       "  'pixel268',\n",
       "  'pixel269',\n",
       "  'pixel270',\n",
       "  'pixel271',\n",
       "  'pixel272',\n",
       "  'pixel273',\n",
       "  'pixel274',\n",
       "  'pixel275',\n",
       "  'pixel276',\n",
       "  'pixel277',\n",
       "  'pixel278',\n",
       "  'pixel279',\n",
       "  'pixel280',\n",
       "  'pixel281',\n",
       "  'pixel282',\n",
       "  'pixel283',\n",
       "  'pixel284',\n",
       "  'pixel285',\n",
       "  'pixel286',\n",
       "  'pixel287',\n",
       "  'pixel288',\n",
       "  'pixel289',\n",
       "  'pixel290',\n",
       "  'pixel291',\n",
       "  'pixel292',\n",
       "  'pixel293',\n",
       "  'pixel294',\n",
       "  'pixel295',\n",
       "  'pixel296',\n",
       "  'pixel297',\n",
       "  'pixel298',\n",
       "  'pixel299',\n",
       "  'pixel300',\n",
       "  'pixel301',\n",
       "  'pixel302',\n",
       "  'pixel303',\n",
       "  'pixel304',\n",
       "  'pixel305',\n",
       "  'pixel306',\n",
       "  'pixel307',\n",
       "  'pixel308',\n",
       "  'pixel309',\n",
       "  'pixel310',\n",
       "  'pixel311',\n",
       "  'pixel312',\n",
       "  'pixel313',\n",
       "  'pixel314',\n",
       "  'pixel315',\n",
       "  'pixel316',\n",
       "  'pixel317',\n",
       "  'pixel318',\n",
       "  'pixel319',\n",
       "  'pixel320',\n",
       "  'pixel321',\n",
       "  'pixel322',\n",
       "  'pixel323',\n",
       "  'pixel324',\n",
       "  'pixel325',\n",
       "  'pixel326',\n",
       "  'pixel327',\n",
       "  'pixel328',\n",
       "  'pixel329',\n",
       "  'pixel330',\n",
       "  'pixel331',\n",
       "  'pixel332',\n",
       "  'pixel333',\n",
       "  'pixel334',\n",
       "  'pixel335',\n",
       "  'pixel336',\n",
       "  'pixel337',\n",
       "  'pixel338',\n",
       "  'pixel339',\n",
       "  'pixel340',\n",
       "  'pixel341',\n",
       "  'pixel342',\n",
       "  'pixel343',\n",
       "  'pixel344',\n",
       "  'pixel345',\n",
       "  'pixel346',\n",
       "  'pixel347',\n",
       "  'pixel348',\n",
       "  'pixel349',\n",
       "  'pixel350',\n",
       "  'pixel351',\n",
       "  'pixel352',\n",
       "  'pixel353',\n",
       "  'pixel354',\n",
       "  'pixel355',\n",
       "  'pixel356',\n",
       "  'pixel357',\n",
       "  'pixel358',\n",
       "  'pixel359',\n",
       "  'pixel360',\n",
       "  'pixel361',\n",
       "  'pixel362',\n",
       "  'pixel363',\n",
       "  'pixel364',\n",
       "  'pixel365',\n",
       "  'pixel366',\n",
       "  'pixel367',\n",
       "  'pixel368',\n",
       "  'pixel369',\n",
       "  'pixel370',\n",
       "  'pixel371',\n",
       "  'pixel372',\n",
       "  'pixel373',\n",
       "  'pixel374',\n",
       "  'pixel375',\n",
       "  'pixel376',\n",
       "  'pixel377',\n",
       "  'pixel378',\n",
       "  'pixel379',\n",
       "  'pixel380',\n",
       "  'pixel381',\n",
       "  'pixel382',\n",
       "  'pixel383',\n",
       "  'pixel384',\n",
       "  'pixel385',\n",
       "  'pixel386',\n",
       "  'pixel387',\n",
       "  'pixel388',\n",
       "  'pixel389',\n",
       "  'pixel390',\n",
       "  'pixel391',\n",
       "  'pixel392',\n",
       "  'pixel393',\n",
       "  'pixel394',\n",
       "  'pixel395',\n",
       "  'pixel396',\n",
       "  'pixel397',\n",
       "  'pixel398',\n",
       "  'pixel399',\n",
       "  'pixel400',\n",
       "  'pixel401',\n",
       "  'pixel402',\n",
       "  'pixel403',\n",
       "  'pixel404',\n",
       "  'pixel405',\n",
       "  'pixel406',\n",
       "  'pixel407',\n",
       "  'pixel408',\n",
       "  'pixel409',\n",
       "  'pixel410',\n",
       "  'pixel411',\n",
       "  'pixel412',\n",
       "  'pixel413',\n",
       "  'pixel414',\n",
       "  'pixel415',\n",
       "  'pixel416',\n",
       "  'pixel417',\n",
       "  'pixel418',\n",
       "  'pixel419',\n",
       "  'pixel420',\n",
       "  'pixel421',\n",
       "  'pixel422',\n",
       "  'pixel423',\n",
       "  'pixel424',\n",
       "  'pixel425',\n",
       "  'pixel426',\n",
       "  'pixel427',\n",
       "  'pixel428',\n",
       "  'pixel429',\n",
       "  'pixel430',\n",
       "  'pixel431',\n",
       "  'pixel432',\n",
       "  'pixel433',\n",
       "  'pixel434',\n",
       "  'pixel435',\n",
       "  'pixel436',\n",
       "  'pixel437',\n",
       "  'pixel438',\n",
       "  'pixel439',\n",
       "  'pixel440',\n",
       "  'pixel441',\n",
       "  'pixel442',\n",
       "  'pixel443',\n",
       "  'pixel444',\n",
       "  'pixel445',\n",
       "  'pixel446',\n",
       "  'pixel447',\n",
       "  'pixel448',\n",
       "  'pixel449',\n",
       "  'pixel450',\n",
       "  'pixel451',\n",
       "  'pixel452',\n",
       "  'pixel453',\n",
       "  'pixel454',\n",
       "  'pixel455',\n",
       "  'pixel456',\n",
       "  'pixel457',\n",
       "  'pixel458',\n",
       "  'pixel459',\n",
       "  'pixel460',\n",
       "  'pixel461',\n",
       "  'pixel462',\n",
       "  'pixel463',\n",
       "  'pixel464',\n",
       "  'pixel465',\n",
       "  'pixel466',\n",
       "  'pixel467',\n",
       "  'pixel468',\n",
       "  'pixel469',\n",
       "  'pixel470',\n",
       "  'pixel471',\n",
       "  'pixel472',\n",
       "  'pixel473',\n",
       "  'pixel474',\n",
       "  'pixel475',\n",
       "  'pixel476',\n",
       "  'pixel477',\n",
       "  'pixel478',\n",
       "  'pixel479',\n",
       "  'pixel480',\n",
       "  'pixel481',\n",
       "  'pixel482',\n",
       "  'pixel483',\n",
       "  'pixel484',\n",
       "  'pixel485',\n",
       "  'pixel486',\n",
       "  'pixel487',\n",
       "  'pixel488',\n",
       "  'pixel489',\n",
       "  'pixel490',\n",
       "  'pixel491',\n",
       "  'pixel492',\n",
       "  'pixel493',\n",
       "  'pixel494',\n",
       "  'pixel495',\n",
       "  'pixel496',\n",
       "  'pixel497',\n",
       "  'pixel498',\n",
       "  'pixel499',\n",
       "  'pixel500',\n",
       "  'pixel501',\n",
       "  'pixel502',\n",
       "  'pixel503',\n",
       "  'pixel504',\n",
       "  'pixel505',\n",
       "  'pixel506',\n",
       "  'pixel507',\n",
       "  'pixel508',\n",
       "  'pixel509',\n",
       "  'pixel510',\n",
       "  'pixel511',\n",
       "  'pixel512',\n",
       "  'pixel513',\n",
       "  'pixel514',\n",
       "  'pixel515',\n",
       "  'pixel516',\n",
       "  'pixel517',\n",
       "  'pixel518',\n",
       "  'pixel519',\n",
       "  'pixel520',\n",
       "  'pixel521',\n",
       "  'pixel522',\n",
       "  'pixel523',\n",
       "  'pixel524',\n",
       "  'pixel525',\n",
       "  'pixel526',\n",
       "  'pixel527',\n",
       "  'pixel528',\n",
       "  'pixel529',\n",
       "  'pixel530',\n",
       "  'pixel531',\n",
       "  'pixel532',\n",
       "  'pixel533',\n",
       "  'pixel534',\n",
       "  'pixel535',\n",
       "  'pixel536',\n",
       "  'pixel537',\n",
       "  'pixel538',\n",
       "  'pixel539',\n",
       "  'pixel540',\n",
       "  'pixel541',\n",
       "  'pixel542',\n",
       "  'pixel543',\n",
       "  'pixel544',\n",
       "  'pixel545',\n",
       "  'pixel546',\n",
       "  'pixel547',\n",
       "  'pixel548',\n",
       "  'pixel549',\n",
       "  'pixel550',\n",
       "  'pixel551',\n",
       "  'pixel552',\n",
       "  'pixel553',\n",
       "  'pixel554',\n",
       "  'pixel555',\n",
       "  'pixel556',\n",
       "  'pixel557',\n",
       "  'pixel558',\n",
       "  'pixel559',\n",
       "  'pixel560',\n",
       "  'pixel561',\n",
       "  'pixel562',\n",
       "  'pixel563',\n",
       "  'pixel564',\n",
       "  'pixel565',\n",
       "  'pixel566',\n",
       "  'pixel567',\n",
       "  'pixel568',\n",
       "  'pixel569',\n",
       "  'pixel570',\n",
       "  'pixel571',\n",
       "  'pixel572',\n",
       "  'pixel573',\n",
       "  'pixel574',\n",
       "  'pixel575',\n",
       "  'pixel576',\n",
       "  'pixel577',\n",
       "  'pixel578',\n",
       "  'pixel579',\n",
       "  'pixel580',\n",
       "  'pixel581',\n",
       "  'pixel582',\n",
       "  'pixel583',\n",
       "  'pixel584',\n",
       "  'pixel585',\n",
       "  'pixel586',\n",
       "  'pixel587',\n",
       "  'pixel588',\n",
       "  'pixel589',\n",
       "  'pixel590',\n",
       "  'pixel591',\n",
       "  'pixel592',\n",
       "  'pixel593',\n",
       "  'pixel594',\n",
       "  'pixel595',\n",
       "  'pixel596',\n",
       "  'pixel597',\n",
       "  'pixel598',\n",
       "  'pixel599',\n",
       "  'pixel600',\n",
       "  'pixel601',\n",
       "  'pixel602',\n",
       "  'pixel603',\n",
       "  'pixel604',\n",
       "  'pixel605',\n",
       "  'pixel606',\n",
       "  'pixel607',\n",
       "  'pixel608',\n",
       "  'pixel609',\n",
       "  'pixel610',\n",
       "  'pixel611',\n",
       "  'pixel612',\n",
       "  'pixel613',\n",
       "  'pixel614',\n",
       "  'pixel615',\n",
       "  'pixel616',\n",
       "  'pixel617',\n",
       "  'pixel618',\n",
       "  'pixel619',\n",
       "  'pixel620',\n",
       "  'pixel621',\n",
       "  'pixel622',\n",
       "  'pixel623',\n",
       "  'pixel624',\n",
       "  'pixel625',\n",
       "  'pixel626',\n",
       "  'pixel627',\n",
       "  'pixel628',\n",
       "  'pixel629',\n",
       "  'pixel630',\n",
       "  'pixel631',\n",
       "  'pixel632',\n",
       "  'pixel633',\n",
       "  'pixel634',\n",
       "  'pixel635',\n",
       "  'pixel636',\n",
       "  'pixel637',\n",
       "  'pixel638',\n",
       "  'pixel639',\n",
       "  'pixel640',\n",
       "  'pixel641',\n",
       "  'pixel642',\n",
       "  'pixel643',\n",
       "  'pixel644',\n",
       "  'pixel645',\n",
       "  'pixel646',\n",
       "  'pixel647',\n",
       "  'pixel648',\n",
       "  'pixel649',\n",
       "  'pixel650',\n",
       "  'pixel651',\n",
       "  'pixel652',\n",
       "  'pixel653',\n",
       "  'pixel654',\n",
       "  'pixel655',\n",
       "  'pixel656',\n",
       "  'pixel657',\n",
       "  'pixel658',\n",
       "  'pixel659',\n",
       "  'pixel660',\n",
       "  'pixel661',\n",
       "  'pixel662',\n",
       "  'pixel663',\n",
       "  'pixel664',\n",
       "  'pixel665',\n",
       "  'pixel666',\n",
       "  'pixel667',\n",
       "  'pixel668',\n",
       "  'pixel669',\n",
       "  'pixel670',\n",
       "  'pixel671',\n",
       "  'pixel672',\n",
       "  'pixel673',\n",
       "  'pixel674',\n",
       "  'pixel675',\n",
       "  'pixel676',\n",
       "  'pixel677',\n",
       "  'pixel678',\n",
       "  'pixel679',\n",
       "  'pixel680',\n",
       "  'pixel681',\n",
       "  'pixel682',\n",
       "  'pixel683',\n",
       "  'pixel684',\n",
       "  'pixel685',\n",
       "  'pixel686',\n",
       "  'pixel687',\n",
       "  'pixel688',\n",
       "  'pixel689',\n",
       "  'pixel690',\n",
       "  'pixel691',\n",
       "  'pixel692',\n",
       "  'pixel693',\n",
       "  'pixel694',\n",
       "  'pixel695',\n",
       "  'pixel696',\n",
       "  'pixel697',\n",
       "  'pixel698',\n",
       "  'pixel699',\n",
       "  'pixel700',\n",
       "  'pixel701',\n",
       "  'pixel702',\n",
       "  'pixel703',\n",
       "  'pixel704',\n",
       "  'pixel705',\n",
       "  'pixel706',\n",
       "  'pixel707',\n",
       "  'pixel708',\n",
       "  'pixel709',\n",
       "  'pixel710',\n",
       "  'pixel711',\n",
       "  'pixel712',\n",
       "  'pixel713',\n",
       "  'pixel714',\n",
       "  'pixel715',\n",
       "  'pixel716',\n",
       "  'pixel717',\n",
       "  'pixel718',\n",
       "  'pixel719',\n",
       "  'pixel720',\n",
       "  'pixel721',\n",
       "  'pixel722',\n",
       "  'pixel723',\n",
       "  'pixel724',\n",
       "  'pixel725',\n",
       "  'pixel726',\n",
       "  'pixel727',\n",
       "  'pixel728',\n",
       "  'pixel729',\n",
       "  'pixel730',\n",
       "  'pixel731',\n",
       "  'pixel732',\n",
       "  'pixel733',\n",
       "  'pixel734',\n",
       "  'pixel735',\n",
       "  'pixel736',\n",
       "  'pixel737',\n",
       "  'pixel738',\n",
       "  'pixel739',\n",
       "  'pixel740',\n",
       "  'pixel741',\n",
       "  'pixel742',\n",
       "  'pixel743',\n",
       "  'pixel744',\n",
       "  'pixel745',\n",
       "  'pixel746',\n",
       "  'pixel747',\n",
       "  'pixel748',\n",
       "  'pixel749',\n",
       "  'pixel750',\n",
       "  'pixel751',\n",
       "  'pixel752',\n",
       "  'pixel753',\n",
       "  'pixel754',\n",
       "  'pixel755',\n",
       "  'pixel756',\n",
       "  'pixel757',\n",
       "  'pixel758',\n",
       "  'pixel759',\n",
       "  'pixel760',\n",
       "  'pixel761',\n",
       "  'pixel762',\n",
       "  'pixel763',\n",
       "  'pixel764',\n",
       "  'pixel765',\n",
       "  'pixel766',\n",
       "  'pixel767',\n",
       "  'pixel768',\n",
       "  'pixel769',\n",
       "  'pixel770',\n",
       "  'pixel771',\n",
       "  'pixel772',\n",
       "  'pixel773',\n",
       "  'pixel774',\n",
       "  'pixel775',\n",
       "  'pixel776',\n",
       "  'pixel777',\n",
       "  'pixel778',\n",
       "  'pixel779',\n",
       "  'pixel780',\n",
       "  'pixel781',\n",
       "  'pixel782',\n",
       "  'pixel783',\n",
       "  'pixel784'],\n",
       " 'target_names': ['class'],\n",
       " 'DESCR': \"**Author**: Yann LeCun, Corinna Cortes, Christopher J.C. Burges  \\n**Source**: [MNIST Website](http://yann.lecun.com/exdb/mnist/) - Date unknown  \\n**Please cite**:  \\n\\nThe MNIST database of handwritten digits with 784 features, raw data available at: http://yann.lecun.com/exdb/mnist/. It can be split in a training set of the first 60,000 examples, and a test set of 10,000 examples  \\n\\nIt is a subset of a larger set available from NIST. The digits have been size-normalized and centered in a fixed-size image. It is a good database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting. The original black and white (bilevel) images from NIST were size normalized to fit in a 20x20 pixel box while preserving their aspect ratio. The resulting images contain grey levels as a result of the anti-aliasing technique used by the normalization algorithm. the images were centered in a 28x28 image by computing the center of mass of the pixels, and translating the image so as to position this point at the center of the 28x28 field.  \\n\\nWith some classification methods (particularly template-based methods, such as SVM and K-nearest neighbors), the error rate improves when the digits are centered by bounding box rather than center of mass. If you do this kind of pre-processing, you should report it in your publications. The MNIST database was constructed from NIST's NIST originally designated SD-3 as their training set and SD-1 as their test set. However, SD-3 is much cleaner and easier to recognize than SD-1. The reason for this can be found on the fact that SD-3 was collected among Census Bureau employees, while SD-1 was collected among high-school students. Drawing sensible conclusions from learning experiments requires that the result be independent of the choice of training set and test among the complete set of samples. Therefore it was necessary to build a new database by mixing NIST's datasets.  \\n\\nThe MNIST training set is composed of 30,000 patterns from SD-3 and 30,000 patterns from SD-1. Our test set was composed of 5,000 patterns from SD-3 and 5,000 patterns from SD-1. The 60,000 pattern training set contained examples from approximately 250 writers. We made sure that the sets of writers of the training set and test set were disjoint. SD-1 contains 58,527 digit images written by 500 different writers. In contrast to SD-3, where blocks of data from each writer appeared in sequence, the data in SD-1 is scrambled. Writer identities for SD-1 is available and we used this information to unscramble the writers. We then split SD-1 in two: characters written by the first 250 writers went into our new training set. The remaining 250 writers were placed in our test set. Thus we had two sets with nearly 30,000 examples each. The new training set was completed with enough examples from SD-3, starting at pattern # 0, to make a full set of 60,000 training patterns. Similarly, the new test set was completed with SD-3 examples starting at pattern # 35,000 to make a full set with 60,000 test patterns. Only a subset of 10,000 test images (5,000 from SD-1 and 5,000 from SD-3) is available on this site. The full 60,000 sample training set is available.\\n\\nDownloaded from openml.org.\",\n",
       " 'details': {'id': '554',\n",
       "  'name': 'mnist_784',\n",
       "  'version': '1',\n",
       "  'description_version': '2',\n",
       "  'format': 'ARFF',\n",
       "  'creator': ['Yann LeCun', 'Corinna Cortes', 'Christopher J.C. Burges'],\n",
       "  'upload_date': '2014-09-29T03:28:38',\n",
       "  'language': 'English',\n",
       "  'licence': 'Public',\n",
       "  'url': 'https://api.openml.org/data/v1/download/52667/mnist_784.arff',\n",
       "  'parquet_url': 'https://data.openml.org/datasets/0000/0554/dataset_554.pq',\n",
       "  'file_id': '52667',\n",
       "  'default_target_attribute': 'class',\n",
       "  'tag': ['Artificial Intelligence',\n",
       "   'AzurePilot',\n",
       "   'Computer Vision',\n",
       "   'Data Sets',\n",
       "   'Kaggle',\n",
       "   'Machine Learning',\n",
       "   'OpenML-CC18',\n",
       "   'OpenML100',\n",
       "   'study_1',\n",
       "   'study_123',\n",
       "   'study_41',\n",
       "   'study_99',\n",
       "   'vision'],\n",
       "  'visibility': 'public',\n",
       "  'minio_url': 'https://data.openml.org/datasets/0000/0554/dataset_554.pq',\n",
       "  'status': 'active',\n",
       "  'processing_date': '2020-11-20 20:12:09',\n",
       "  'md5_checksum': '0298d579eb1b86163de7723944c7e495'},\n",
       " 'url': 'https://www.openml.org/d/554'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:41:44.204924Z",
     "start_time": "2025-03-12T08:41:44.197818Z"
    }
   },
   "cell_type": "code",
   "source": [
    "X, y = mnist['data'], mnist['target']\n",
    "X.shape"
   ],
   "id": "e416726e63632c02",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000, 784)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:41:45.360826Z",
     "start_time": "2025-03-12T08:41:45.355268Z"
    }
   },
   "cell_type": "code",
   "source": "y.shape",
   "id": "576abc2672a92145",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "![](model/9.png)",
   "id": "af84bd06f53accc2"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:41:49.601299Z",
     "start_time": "2025-03-12T08:41:49.440380Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 取出前60000为训练集，剩余作为测试集\n",
    "X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]\n",
    "# 洗牌操作\n",
    "shuffle_index = np.random.permutation(60000)\n",
    "X_train, y_train = X_train.iloc[shuffle_index], y_train.iloc[shuffle_index]"
   ],
   "id": "c43f7edb86765a18",
   "outputs": [],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:41:49.942255Z",
     "start_time": "2025-03-12T08:41:49.938087Z"
    }
   },
   "cell_type": "code",
   "source": "shuffle_index",
   "id": "5591d2d0a00e061d",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([12628, 37730, 39991, ...,   860, 15795, 56422], dtype=int32)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 7
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "## 交叉验证\n",
    "通过多次划分数据集来减少模型评估的随机性误差，并避免过拟合问题。  \n",
    "训练集-验证集-测试集"
   ],
   "id": "c7312ac3970c0340"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "![](model/5.png)",
   "id": "fd3e6cc85a05498e"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "![](model/6.png)",
   "id": "d18eec20400b6689"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:41:56.898443Z",
     "start_time": "2025-03-12T08:41:56.887925Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 二分类法，将数据集划分为三份\n",
    "# 判断数字是否等于6，等于True，不等于False\n",
    "y_train_6 = (y_train == '6')\n",
    "y_train_6[:10]"
   ],
   "id": "b5191b0b427b0757",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12628    False\n",
       "37730    False\n",
       "39991    False\n",
       "8525     False\n",
       "8279     False\n",
       "51012    False\n",
       "14871    False\n",
       "15127    False\n",
       "9366     False\n",
       "33322    False\n",
       "Name: class, dtype: bool"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:42:07.845134Z",
     "start_time": "2025-03-12T08:42:07.163302Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 训练函数\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "sgd_clf = SGDClassifier(max_iter=5, random_state=42)\n",
    "sgd_clf.fit(X_train, y_train_6)"
   ],
   "id": "d7ca3d866f09d71b",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(max_iter=5, random_state=42)"
      ],
      "text/html": [
       "<style>#sk-container-id-1 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: #000;\n",
       "  --sklearn-color-text-muted: #666;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-1 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-1 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: flex;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "  align-items: start;\n",
       "  justify-content: space-between;\n",
       "  gap: 0.5em;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label .caption {\n",
       "  font-size: 0.6rem;\n",
       "  font-weight: lighter;\n",
       "  color: var(--sklearn-color-text-muted);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-1 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-1 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-1 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 0.5em;\n",
       "  text-align: center;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-1 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>SGDClassifier(max_iter=5, random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow\"><div><div>SGDClassifier</div></div><div><a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.6/modules/generated/sklearn.linear_model.SGDClassifier.html\">?<span>Documentation for SGDClassifier</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></div></label><div class=\"sk-toggleable__content fitted\"><pre>SGDClassifier(max_iter=5, random_state=42)</pre></div> </div></div></div></div>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:42:09.104062Z",
     "start_time": "2025-03-12T08:42:09.098693Z"
    }
   },
   "cell_type": "code",
   "source": "sgd_clf.predict([X.iloc[35000]])",
   "id": "2f37f4bde4655204",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:42:10.767256Z",
     "start_time": "2025-03-12T08:42:10.758960Z"
    }
   },
   "cell_type": "code",
   "source": "y[35000]",
   "id": "3192f90e19eabc6e",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:42:15.481117Z",
     "start_time": "2025-03-12T08:42:13.891644Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 交叉验证函数，三组的准确率\n",
    "from sklearn.model_selection import cross_val_score\n",
    "cross_val_score(sgd_clf, X_train, y_train_6, cv=3, scoring='accuracy')"
   ],
   "id": "edb7d4511277ea3a",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.9821 , 0.9773 , 0.98025])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 12
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## Confusion Matrix 混淆矩阵",
   "id": "a1ab651d6493879a"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "![](model/8.png)",
   "id": "9616e4e6d28ba04d"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:42:21.253796Z",
     "start_time": "2025-03-12T08:42:19.502599Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 划分训练后三组预测值的和\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_6, cv=3)\n",
    "y_train_pred.shape"
   ],
   "id": "be8c642158d6a12e",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:42:21.259801Z",
     "start_time": "2025-03-12T08:42:21.254802Z"
    }
   },
   "cell_type": "code",
   "source": "X_train.shape",
   "id": "9bdf84b2a46169d8",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 784)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:42:22.106484Z",
     "start_time": "2025-03-12T08:42:22.097996Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 混淆矩阵函数\n",
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_train_6, y_train_pred)"
   ],
   "id": "3ee826ba93b9db1d",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[53357,   725],\n",
       "       [  482,  5436]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 15
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "[ **true negatives(TN)** , **false positives(FP)** ]  \n",
    "[ **false negatives(FN)** , **true positives(TP)** ]"
   ],
   "id": "dfb3c6571242c3f5"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "### **Precision and Recall (精度和召回率)**  \n",
    "\n",
    "### **$ precision = \\frac {TP} {TP + FP} $**\n",
    "\n",
    "### **$ recall = \\frac {TP} {TP + FN} $**"
   ],
   "id": "fc6810c9607b4665"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:42:25.940642Z",
     "start_time": "2025-03-12T08:42:25.928381Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "precision_score(y_train_6, y_train_pred)"
   ],
   "id": "de050bbca113f3a9",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8823242980035708"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 16
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:42:27.281684Z",
     "start_time": "2025-03-12T08:42:27.269915Z"
    }
   },
   "cell_type": "code",
   "source": "recall_score(y_train_6, y_train_pred)",
   "id": "542b3ef9b07aa129",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9185535653937141"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 17
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "将 **Precision** 和 **Recall** 结合到一个称为 **F1 score** 的指标,调和平均值给予低值更多权重。  \n",
    "因此，如果召回和精确度都很高，分类器将获得高 F1 分数。\n",
    "\n",
    "### $ F_1  = $ $2\\over {1\\over precision}+{1\\over recall} $ $=$ $2×$ $precision×recall\\over precision+recall $ $=$ $TP\\over {TP}+{FN + FP\\over 2}$"
   ],
   "id": "4658b6411e352050"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:42:31.314430Z",
     "start_time": "2025-03-12T08:42:31.295398Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from sklearn.metrics import f1_score\n",
    "f1_score(y_train_6, y_train_pred)"
   ],
   "id": "65a89d73c760a7e2",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9000745094792615"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 18
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "## 阈值对结果的影响\n",
    "阈值越高，精度越高，召回率越低。  \n",
    "反之，阈值越低，精度越低，召回率越高。"
   ],
   "id": "719be488eb5706a9"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "![](model/2.png)",
   "id": "dc8b0ed68d418e6c"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:44:22.689396Z",
     "start_time": "2025-03-12T08:44:22.677921Z"
    }
   },
   "cell_type": "code",
   "source": [
    "y_scores = sgd_clf.decision_function([X.iloc[6000]])\n",
    "y_scores"
   ],
   "id": "74c0f80bc38407e3",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-5150.84649451])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 25
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T08:44:38.073231Z",
     "start_time": "2025-03-12T08:44:38.068585Z"
    }
   },
   "cell_type": "code",
   "source": [
    "t = -6000\n",
    "y_pred = (y_scores > t)\n",
    "y_pred"
   ],
   "id": "7fa8d32b4d21d9d3",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 26
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "Scikit-Learn 不允许直接设置阈值，但它可以得到决策分数，调用其 **decision_function()** 方法，而不是调用分类器的 **predict()** 方法，该方法返回每个实例的分数，然后使用想要的**阈值**根据这些分数进行预测:",
   "id": "6195affc82f03663"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T09:00:32.085714Z",
     "start_time": "2025-03-12T09:00:30.155317Z"
    }
   },
   "cell_type": "code",
   "source": [
    "y_scores = cross_val_predict(sgd_clf, X_train, y_train_6, cv=3, method='decision_function')\n",
    "y_scores[:10]"
   ],
   "id": "b31e4fa425c68a6c",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1047802.11504311,  -900272.52534366, -1268468.43676609,\n",
       "        -867121.16638167, -1588369.77955884,  -565026.52909493,\n",
       "       -1633300.99812212,  -896855.34416366,  -960405.5608314 ,\n",
       "        -409805.60759782])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 28
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T09:06:39.360798Z",
     "start_time": "2025-03-12T09:06:39.336395Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "precisions, recalls, thresholds = precision_recall_curve(y_train_6, y_scores)"
   ],
   "id": "55807c80a3deaf3e",
   "outputs": [],
   "execution_count": 29
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T09:06:56.744823Z",
     "start_time": "2025-03-12T09:06:56.739131Z"
    }
   },
   "cell_type": "code",
   "source": "y_train_6.shape",
   "id": "854126e4c6f12b67",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 30
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T09:07:03.399595Z",
     "start_time": "2025-03-12T09:07:03.394455Z"
    }
   },
   "cell_type": "code",
   "source": "thresholds.shape",
   "id": "66c40f1fe73e243e",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 31
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T09:07:16.090014Z",
     "start_time": "2025-03-12T09:07:16.082967Z"
    }
   },
   "cell_type": "code",
   "source": "precisions[:10]",
   "id": "6d0da6c482e6a722",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.09863333, 0.09863498, 0.09863662, 0.09863827, 0.09863991,\n",
       "       0.09864155, 0.0986432 , 0.09864484, 0.09864649, 0.09864813])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 32
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T09:07:28.141930Z",
     "start_time": "2025-03-12T09:07:28.135121Z"
    }
   },
   "cell_type": "code",
   "source": "recalls[:10]",
   "id": "3492c2562f93c3f8",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 33
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T09:07:46.433345Z",
     "start_time": "2025-03-12T09:07:46.428897Z"
    }
   },
   "cell_type": "code",
   "source": "precisions.shape",
   "id": "ef5a6bd0871df2e5",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60001,)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 34
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T09:07:57.207437Z",
     "start_time": "2025-03-12T09:07:57.203152Z"
    }
   },
   "cell_type": "code",
   "source": "recalls.shape",
   "id": "6a4dba6a24c20c04",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60001,)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 35
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "## ROC curves\n",
    "**receiver operating characteristic (ROC)** 曲线是二元分类中的常用评估方法\n",
    "- 它与精确度/召回曲线非常相似，但ROC曲线不是绘制精确度与召回率，而是绘制 **true positive rate(TPR)** 与 **false positive rate(FPR)**  \n",
    "- 要绘制ROC曲线，首先需要使用 **roc_curve()** 函数计算各种阈值的 **TPR和FPR** ：  \n",
    "### $ {TPR} = \\frac {TP} {TP + FN} $ (Recall)\n",
    "### $ {FPR} = \\frac {FP} {FP + TN} $"
   ],
   "id": "2513f9dee2f26812"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T09:29:46.263264Z",
     "start_time": "2025-03-12T09:29:46.240338Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "fpr, tpr, thresholds = roc_curve(y_train_6, y_scores)"
   ],
   "id": "39315047c08f1ca9",
   "outputs": [],
   "execution_count": 40
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T09:31:17.744158Z",
     "start_time": "2025-03-12T09:31:17.665024Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0, 1], [0, 1], 'k--')\n",
    "    plt.axis((0, 1, 0, 1))\n",
    "    plt.xlabel('False Positive Rate', fontsize=16)\n",
    "    plt.ylabel('True Positive Rate', fontsize=16)\n",
    "\n",
    "plt.figure(figsize=(6, 6))\n",
    "plot_roc_curve(fpr, tpr)"
   ],
   "id": "8e9b0d65de4cfc05",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 600x600 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAIeCAYAAABz8u9MAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABzxklEQVR4nO3deVxU5eIG8Gdm2HdBEHAXxSU1rRSXEsh9T0Ncc01tUcslzdLUNE3tYpp1y1Dcc7dU3AW87kuilrvggoriAgMIDDDz/v4w5icBOgwznJnh+X4+fC6cOWfmmXPJeXjPe86RCSEEiIiIiMyEXOoARERERMXB8kJERERmheWFiIiIzArLCxEREZkVlhciIiIyKywvREREZFZYXoiIiMissLwQERGRWWF5ISIiIrPC8kJERERmxSTLS3p6OqZNm4YOHTrA3d0dMpkMy5cv13n7lJQUjBgxAp6ennB0dERwcDDOnDljvMBERERUakyyvDx69Ahff/01Ll26hFdffbVY22o0GnTu3Blr167FqFGjMG/ePCQlJSEoKAjXrl0zUmIiIiIqLVZSByiMj48PEhMT4e3tjdOnT6NJkyY6b7tp0yYcPXoUGzduREhICAAgNDQU/v7+mDZtGtauXWus2ERERFQKTHLkxdbWFt7e3nptu2nTJlSoUAE9e/bULvP09ERoaCj++OMPqFQqQ8UkIiIiCZhkeSmJ2NhYvPbaa5DL87+1pk2bIiMjA1evXpUoGRERERmCSR42KonExES0atWqwHIfHx8AwL1799CgQYNCt1WpVPlGZjQaDZ48eQIPDw/IZDLjBCYiIrJAQgikpaXB19e3wIBCSVlcecnMzIStrW2B5XZ2dtrHizJnzhzMmDHDaNmIiIjKmoSEBFSqVMmgz2lx5cXe3r7QeS1ZWVnax4syefJkjBs3TvuzUqlElSpVkJCQABcXF8OHJfqHMjMHObkaCCGgFgIaAWg0AhoIqDUCSakqQAAqjQZRlx7A0dYKas2zv2xyNRqoNcCjdBWuJaVBLpMh4UkmfN3skKsWUGs0ePw0R+q3aFacbBWQy2RQyGWQy2TIUWuQmpWLV3xdIJPJIJfh2eMyGWT/fK/MyoEqR436FV3/2fbZcrn82fr3UjJRrbwTXO2sAQB5f4jK/3kO2XPf/3t5SkYuang6wkohh5X8n1xyGdQaDbycbSFD0SPDugwav2ydFz2/7s+hg5eu9OIVdHqvL3tchyd5+XPokuMl7+XlT/Hy13jZ/ydGeK/Tv5yE31atgEwmw9fffoepk8bD2dn5pa9TXBZXXvLOVPq3vGW+vr5Fbmtra1voqI2LiwvLSxkl8oqEeFYihADUQkCVo8b1pPRnpUMtcPl+KuxtFMjK0eBY3CNU83BErkYgR63B7r/vo35FV2iEwOP0bNxNyYSrvTVy1RrkaASyczUGTPzsE1Fu64D72kFGBeS21gZ8jZJztrOCtUKOJ0+zUcfbGRVc7LQfyo+fZsPWSo6aXk7aD3D5c4VBpl0G3E3ORP2KrrCxkj9bjn+v+///m6PWoLK7A6z/KQD2Ngq42dtAIZfB6p8yYGslh521QurdQ2S2+vZ6F79v2oDw8HB06dIFUyeNN8q0C4srL40aNcKhQ4eg0WjyHWM7ceIEHBwc4O/vL2E6MrasHDVUORqotSMSzwpDVo4ad5Izka3W4ET8E1x/mI4KzrZQa56NdKg1AoeuPUKN8o5IV+UiKU0FmQwQQr8cp24m5/v5bEJKvp+VmaUzElLBxRZWcjmsFTLcfJyByu72qFzOQTuioJA/+0flYZoKVTwc8FqVcrCzlkMIwNZKjvJOts/++n+uGOSNBsie+14ul/2zDABksLdWwMXeCgr5s9EJxT/FxNH2WWkhIsvUtWtXxMfHw9vbG6mpqUZ7HbMuL4mJiVAqlfDz84O19bO/LENCQrBp0yZs2bJFe52XR48eYePGjejatWuhIyskjVy1BklpKmTnapCr0eBRejYyc9TIylbj6oN0ONtZQa0R2hGMEzceo6qHI3LVGpy/o4RGCFx9kA53RxsIIZCcUfJCEP/oqfZ7fYvLi9hYyWEtl+FpthqV3e3hZGsNK7kM91Iy4WJvDf8KTs8dang2gqCQyaDMzIGdjQK1vJxgrZBDlaNGK39P7WEEubYgAF4udnCwVsCKJYGIjEytVuPzzz/Hhx9+iBo1agCA3pc6KQ6TLS+LFy9GSkoK7t27BwDYvn077ty5AwAYPXo0XF1dMXnyZKxYsQI3btxAtWrVADwrL82aNcOQIUNw8eJFlC9fHj/99BPUajUn45ayjOxcXEpMxembyYi9nYKbj5/i8v00AM+G/DV6lIMj1x8XWPbkaXZJoxbg42qHRGUWqpd3RDkHayjkzw5XKGQyyP+Zy3A3JRNezrYI9PeCg40CGdlq+Fdwgq2VAvY2Crg72sBKLoO1Qg5Xe2vY2/BwBBFZDrVajSFDhmDVqlXYsmULLl68WGoDBCZbXr777jvcunVL+/OWLVuwZcsWAMCAAQPg6upa6HYKhQI7d+7EZ599hkWLFiEzMxNNmjTB8uXLUbt27VLJbqmEELj9JANpWbk4ePUhhBBIVGbhWlI6UjNzkJyRjfJOtrj1OAPpqtwXPpc+xeVlank5ITUrB09VagRUd382l0EhgxBAojILb9Ysj+SMbDSo6ApnO2vU83WBi50VrORyKBTP5j3YKOSQy3laPBHRi6jVagwaNAhr1qyBlZUV5s2bV6pHNmRCGGNw3DKkpqbC1dUVSqWyTEzYzcxW454yE6ocDRKVmfj7bir+vJ2M2NvJSMt6cRnRx+tVy+FuciYqlbNHtfKOsFbIcPtJBppV94CNlRy5GoHq5R3zT6hUyOHpbAsrhRx21nJUcLZj2SAiKkW5ubkYNGgQ1q5dCysrK6xfvz7fVe3zGPMz1GRHXshwNBqBP28n41xCCtT/nN3yx7l7yFFrUN7JFskZ2Yh/+PTlT6QjBxsFbKzkSMnIgbOdFV6vWg7Na3igYSU3NKjkCidb/toREZmj3NxcDBw4EL/99husrKywYcMG9OjRo9Rz8FPEAmXnanD7SQaW/C8OG07feeG6tx5nFPv5X63kiqbV3ZGuykWnBj6wksvh7WoHN3trlHO00Tc2ERGZuOnTp+O3336DtbU1Nm7ciO7du0uSg+XFjAkh8PfdVNx+koHY28nYcDoBqXoe3rG3ViBbrYGbvTWa1fCAp7MtHj/NxutV3PCWvyeq/HN9DCIiKrs+/fRT7NmzB1OnTkW3bt0ky8HyYkbuJGdgw+k7OHr9EU7fSn75Bv9Su4IzOjf0gX8FZzj8czZMHW9nnlJLRERFEkJoLzRXvnx5nDhxwuD3KioulhcTlZaVg4v3UpGckY2lh28UuOiZLkLfqISqHo4Y9mZ1XjWUiIiKLScnB/369UObNm0wcuRIAJC8uAAsLybn9uMMtJofXaxtKrrZo0+TyvBxs0erWuXh5WJnpHRERFRWZGdno0+fPti6dSu2b9+Ozp07G/wGi/pieTEB607expJD8Tqf8TOlc128Was8/DydOA+FiIgMLjs7G71798bvv/8OW1tbbN261WSKC8DyIpmnqlyM33AOuy/cf+F67zWrCm9XO1R2d0Cn+t6cn0JEREaVnZ2N0NBQ/PHHH7C1tcXvv/+ODh06SB0rH5aXUrY19g7Grj/3wnVsrOQIC30VXRoWfQdsIiIiQ1OpVOjVqxe2b98OW1tb/PHHH2jfvr3UsQpgeSkFQgisOn4LX/1xoch1qno4YMPI5qjA+SpERCSRzZs3Y/v27bCzs8Mff/yBdu3aSR2pUCwvpWDUb7GIPJ9Y6GPj2vrj4+CaUPAS90REJLG+ffsiLi4OzZs3R5s2baSOUyTe2+gFSnpfhtjbyejx09FCHzs0MRiV3R1KGpGIiKhEsrKyoFar4ejoaNDn5b2NzExSahaazj5Q6GMXZrSHI+/tQ0REJiArKws9evRAVlYWIiMj4eBgHn9U81PUwDp8/z9cvp9WYHlNLyfsGP0mLxZHREQmISsrC++88w727NkDe3t7XLhwAU2aNJE6lk5YXgxECIFGX++DMjOnwGM7Rr+J+hVdJUhFRERUUGZmJt555x3s3bsXDg4OiIyMNJviArC8GEzrsIMFisuS915Hu1e8JUpERERUUEZGBrp37479+/fDwcEBO3fuRGBgoNSxioXlxQA+33y+wNVxb8zppL2RFRERkSnIyMhAt27dcODAATg6OmLnzp1o1aqV1LGKjeWlhMIPxWPdqYR8y1hciIjIFN26dQtnzpyBo6Mjdu3ahbfeekvqSHpheSmhJf+Lz/dz3GwWFyIiMk1169bF/v37kZGRgTfffFPqOHpjeSmBtKwcJKWptD9fntmBF5sjIiKT8vTpU1y7dg2NGjUCALz22mvSBjIA3uWvBH6KidN+/4qvC0+DJiIik/L06VN07twZrVq1wrFjx6SOYzAsLyWw5cwd7fdDWlaXMAkREVF+6enp6NSpEw4ePAiZTGZRUxp42EhPp28+wYPU/z9k9O5rFSVMQ0RE9P/yisuhQ4fg4uKCvXv3IiAgQOpYBsORFz0IIdA//IT253Ft/S2q0RIRkflKS0tDx44dcejQIbi6umLfvn0WVVwAjrzo5fStZKhyNQAANwdrfBDoJ3EiIiKi/y8uR44c0RYXc7pyrq448qKHbWfvab/vWN8HNlbcjUREJD1ra2u4uLjAzc0N+/fvt8jiAnDkpdiyctRYdfyW9udhb1aTLgwREdFz7OzssGXLFty4cQN169aVOo7RcMigmI7HP9Z+X87BGjW9nCVMQ0REZZ1SqcSiRYsghADwrMBYcnEBOPJSbM/fw6hlzfISJiEiorIuJSUF7du3x8mTJ/H48WPMmDFD6kilguWlmObtuaz9vnMDHwmTEBFRWZaSkoJ27drh1KlT8PDwQM+ePaWOVGp42KiYsnI02u9fr1ZOwiRERFRWJScno23btjh16hTKly+PqKgovPrqq1LHKjUceSmGzGx1vp+9nO0kSkJERGVVXnH5888/tcWlQYMGUscqVRx5KYYL95Ta7z2dbSVMQkREZZFarUb79u3x559/wtPTE9HR0WWuuAAsL8Wy58J97fcdXvGWMAkREZVFCoUCY8aMgbe3N6KiolC/fn2pI0mC5aUYYq481H7PM42IiEgKAwYMwLVr18pscQFYXorlWlK69vtW/iwvRERkfI8ePUKvXr2QmJioXebk5CRhIulxwq6OVLn5J+s62HDXERGRcT18+BCtW7fGX3/9hcePHyMqKkrqSCaBIy86SkzJ0n7v7mgjYRIiIioLkpKS8Pbbb+Ovv/6Ct7c3fvrpJ6kjmQwOH+jonjJT+71/hbI9XEdERMaVV1wuXLgAHx8fREdHo3bt2lLHMhkcedFRwpMM7fct/DjfhYiIjOPBgwcIDg7GhQsX4Ovri5iYGBaXf2F50dHNx/9fXmp5ceSFiIiMY/jw4bh48SIqVqyImJgY+Pv7Sx3J5PCwkY42nErQfl+Lh42IiMhIfvrpJ6SlpeHXX39FzZo1pY5jklhedFTeyRaPn2YDAKqXZ3khIiLDycnJgbW1NQCgUqVKiI6OljiRaeNhIx3FP/r/a7wo5DIJkxARkSW5d+8eGjdujA0bNkgdxWywvOggMSUTOWoBAGhWw13iNEREZCnu3bunnZw7efJkqFQqqSOZBZYXHSQ8+f/TpG2sFBImISIiS3H37l0EBQXh6tWrqFq1Kvbv3w9bW970Vxec86KD+6n/X15eq+ImXRAiIrIId+7cQXBwMK5fv46qVasiJiYG1apVkzqW2eDIiw7up/7/1XVrV3CWMAkREZm7hIQEBAUF4fr166hWrRoOHjzI4lJMLC86SM3K1X5fjrcGICKiEli+fDni4uJQvXp1HDx4EFWrVpU6ktnhYSMdKDOytd+Xc2B5ISIi/U2ZMgVCCAwePBhVqlSROo5ZYnnRwYV7qdrvXey5y4iIqHju3buH8uXLw8bGBjKZDF999ZXUkcwaDxvpIOW5kZfyTpwJTkREurt58yZatmyJ0NBQZGdnv3wDeikOI+ggKS0bcttnu8pawb5HRES6uXnzJoKCgnDr1i1YW1sjOTkZFSpUkDqW2eMncTF4OnPUhYiIdHPjxg0EBgbi1q1b8Pf3R3R0NIuLgXDkpRgepvHKh0RE9HLx8fEICgpCQkKCtrj4+vpKHcticOSlGFrX8ZI6AhERmbi4uDgEBgYiISEBtWvXRkxMDIuLgbG8FIOXCw8bERHRiyUmJuLJkyeoU6cOYmJi4OPjI3Uki8PDRsWQlMrDRkRE9GJvvvkm9u3bhxo1asDb21vqOBaJ5aUYano5SR2BiIhM0NWrV5GVlYWGDRsCAFq0aCFxIsvGw0bFUNub9zUiIqL8rly5gqCgILRu3RoXLlyQOk6ZwPJSDN4udlJHICIiE3L58mUEBwcjMTER3t7e8PLiiR2lgeWlGKytuLuIiOiZ54tLgwYNEBUVBU9PT6ljlQn8NC4G3pSRiIgA4NKlSwgKCsL9+/fRsGFDFpdSxgm7xWDLkRciojLv6tWrCAoKQlJSEl599VUcOHAAHh4eUscqU1heisHWmuWFiKis8/HxQa1ateDr64v9+/ezuEiA5aUYHG24u4iIyjpnZ2fs2rULOTk5cHd3lzpOmcShhGLgYSMiorLp/PnzWLBggfZnZ2dnFhcJcShBR1ZyGawULC9ERGXNuXPn0Lp1azx+/Bhubm4YMmSI1JHKPH4a68jRlj2PiKisOXv2rLa4vPHGG+jRo4fUkQgsLzpzsFFIHYGIiEpRbGystrg0adIE+/btg5ubm9SxCCwvOrNneSEiKjPOnDmD1q1b48mTJ2jatCmLi4lhedERzzQiIiobHj9+jLZt2yI5ORkBAQHYu3cvXF1dpY5Fz2F50ZGVQiZ1BCIiKgUeHh6YNm0amjdvzuJiolhedGQt564iIiorxowZg4MHD8LFxUXqKFQIfiLryNqKIy9ERJbq1KlTaNOmDZKTk7XLrK2tJUxEL8LyoqNEZZbUEYiIyAhOnDiBNm3a4MCBA5gyZYrUcUgHLC868nDkHaWJiCzN8ePH0a5dO6SmpuKtt97C3LlzpY5EOmB50VEFFzupIxARkQEdO3ZMW1wCAwOxc+dOODk5SR2LdGCy5UWlUmHSpEnw9fWFvb09AgICsG/fPp223b9/P4KDg1G+fHm4ubmhadOmWLVqVYnyPE7PLtH2RERkOo4cOYJ27dohLS0NQUFBiIyMZHExIyZbXgYPHoywsDD0798fCxcuhEKhQKdOnXD48OEXbrdt2za0a9cO2dnZmD59Or755hvY29tj4MCB+W6qVVwNK/FUOSIiS6BWqzFs2DCkp6cjODgYO3bsgKOjo9SxqBhkQgghdYh/O3nyJAICAjB//nxMmDABAJCVlYX69evDy8sLR48eLXLbdu3a4cKFC4iPj4etrS0AIDc3F3Xq1IGjoyPOnTunc47U1FS4urqi8qcb8EnHhhjfrnbJ3hgREZmEq1evYsaMGfj111/h4OAgdRyLlPcZqlQqDX7KuUmOvGzatAkKhQIjRozQLrOzs8OwYcNw7NgxJCQkFLltamoqypUrpy0uAGBlZYXy5cvD3t5e70x3kzP13paIiKSXlpam/d7f3x9r1qxhcTFTJlleYmNj4e/vX6CpNW3aFMCzu3wWJSgoCBcuXMDUqVNx/fp1xMXFYebMmTh9+jQmTpyod6aaFXgslIjIXB08eBDVq1fHnj17pI5CBmCSN+xJTEyEj49PgeV5y+7du1fktlOnTsWNGzfwzTffYNasWQAABwcHbN68Gd27d3/h66pUKqhUKu3Pqamp2u9tFCbZ84iI6CViYmLQuXNnZGRk4KeffkL79u2ljkQlZJKfyJmZmfkO++Sxs7PTPl4UW1tb+Pv7IyQkBL/99htWr16NN954AwMGDMDx48df+Lpz5syBq6ur9qty5crax6xZXoiIzE50dLS2uLRv3x7r16+XOhIZgEmOvNjb2+cbAcmTlZWlfbwoo0aNwvHjx3HmzBnI/7kfUWhoKF555RV88sknOHHiRJHbTp48GePGjdP+nJqaqi0wCjlvD0BEZE6ioqLQpUsXZGZmokOHDti6dav2j2AybyY5nODj44PExMQCy/OW+fr6FrpddnY2li5dis6dO2uLC/Ds/hQdO3bE6dOnkZ1d9PVabG1t4eLiku+LiIjMz4EDB7TFpVOnTiwuFsYky0ujRo1w9erVfHNOAGhHTRo1alTodo8fP0Zubi7UanWBx3JycqDRaAp9TBdZOfptR0REpe+3335DZmYmOnfujC1btrC4WBiTLC8hISFQq9VYsmSJdplKpUJERAQCAgK0h3Ju376Ny5cva9fx8vKCm5sbtm7dmm+EJT09Hdu3b0edOnX0Pl3a25W/+ERE5uLnn39GWFgYNm/eXOgcSjJvJjnnJSAgAL169cLkyZORlJSEmjVrYsWKFbh58yaWLl2qXW/gwIE4ePAg8q6zp1AoMGHCBEyZMgXNmjXDwIEDoVarsXTpUty5cwerV6/WO5OV3CR7HhER/ePcuXOoX78+FAoFrKysMHbsWKkjkZGY7CfyypUr8emnn2LVqlUYM2YMcnJysGPHDrRq1eqF23355ZdYs2YNrK2tMWPGDEydOhUuLi7YtGkT+vfvr3ceGytO2CUiMlW7d+9GQEAAhg0bpvf0ADIfJnl7AFPx/O0Blg5vhbb1KkgdiYiI/mXnzp3o0aMHsrOz0b17d2zYsAE2NjZSxyrzytztAUyRnTV3FRGRqYmMjNQWlx49erC4lBH8RNaRrZVC6ghERPScHTt2oGfPnsjOzsa7776L9evXs7iUESwvOuIFdomITMf27du1xSXviurW1tZSx6JSwo9kHSl4thERkcmQyZ6dRBEaGoq1a9eyuJQxJnmqtClSyHi2ERGRqejSpQsOHz6M1157DVZW/CgrazicoCMOvBARSWvHjh2Ii4vT/ty0aVMWlzKKH8k64kXqiIiks3nzZvTo0QPBwcG4d++e1HFIYvxE1pEAL4dDRCSFTZs2oXfv3sjNzcVbb70FLy8vqSORxFhedGTHU6WJiErdxo0b0adPH6jVagwYMAArV67koSJiedGVQs4Ju0REpWn9+vXo27cv1Go13nvvPSxfvhwKBf+QJJYXnbG8EBGVnsjISPTv3x9qtRqDBg1CREQEiwtpcexNRywvRESlp1mzZqhfvz4aN26M8PBwFhfKh+VFRywvRESlx8PDAzExMXBxcYGcZ3vSv/A3Qke8SB0RkXGtXr0aP//8s/ZnNzc3FhcqFEdedKRQsLwQERnLqlWrMGjQIAghULduXQQGBkodiUwYK62OOPJCRGQcK1as0BaXkSNH4q233pI6Epk4lhcdcc4LEZHhLV++HEOGDIEQAh9++CF++uknHiqil+JviI5YXoiIDCsiIgJDhw6FEAIfffQRfvzxRxYX0gl/S3Qk52EjIiKDiY2NxbBhwyCEwKhRo7B48WLI+O8s6YgTdnXEgRciIsNp3Lgxpk6diuTkZCxcuJDFhYqF5UUHMhn4HxYRkQGo1WrtBeemT58OgP++UvHxsJEOeMiIiKjkfv75Z7Rt2xZPnz4F8Ky0sLiQPlhedMBDRkREJfPf//4XH374IaKjo7Fq1Sqp45CZY3nRAf8yICLS348//oiPPvoIADBhwgSMHDlS4kRk7lhedMCRFyIi/SxevBijRo0CAHz22WeYN28e/yCkEmN50QHLCxFR8S1atAijR48GAEyaNAlz585lcSGDYHkhIiKDe/z4MWbMmAEA+PzzzzFnzhwWFzIYniqtg4xsjdQRiIjMioeHB/bv34+dO3fiiy++YHEhg2J50UF5JxupIxARmYXExET4+PgAeHYhusaNG0uciCwRDxvpwIqTXoiIXuq7775D7dq1cfToUamjkIVjedGBQsHyQkT0IvPmzcNnn32GtLQ0xMTESB2HLBzLiw6seJdTIqIizZ07F5MmTQLw7JL/X3zxhcSJyNLxU1kHCh42IiIq1Jw5c/D5558DAGbMmIFp06ZJnIjKghJP2FUqlTh16hQePnyIqlWrokWLFobIZVISkjOljkBEZHK++eYbTJkyBQAwc+ZM7fdExqb3yEtaWhref/99eHl5oX379hgwYADCw8O1j4eHh8PX1xcnTpwwSFApcdyFiCg/tVqtnZj7fIkhKg16lZfMzEwEBQVh2bJlKFeuHDp27AghRL51unTpggcPHuD33383RE5J1fJykjoCEZFJUSgU2Lx5M9avX885LlTq9CovYWFhiI2NRd++fREXF4cdO3YUWMfb2xt169ZFdHR0iUNKTc6LKxERAQD27t2r/WPVzs4OoaGhEieiskiv8rJ+/Xp4e3tj6dKlcHR0LHI9f39/3LlzR+9wpoITdomInp1J1L59e4wfP77AaDtRadKrvMTFxaFp06aws7N74XoODg549OiRXsFMiYIjL0RUhgkhMG3aNO29inx9fXm5f5KUXmcbKRQK5OTkvHS9O3fuvHBkxlzwMi9EVFYJIfDVV19h1qxZAJ5dRXf8+PESp6KyTq+PZT8/P5w7dw65ublFrpOeno7z58+jbt26eoczFZzzQkRlkRACU6ZM0RaXsLAwFhcyCXqVl27duiExMVH7C12YWbNmQalUokePHnqHMxVyznkhojJoypQpmD17NgBgwYIFGDt2rMSJiJ7Rq7yMHTsWFStWxMyZM/HOO+9g7dq1AIAHDx5gy5Yt6NOnD+bPn49q1arhgw8+MGhgIiIqHXXr1oVcLsfChQvx6aefSh2HSEsm9JwyfvHiRXTr1g3x8fEFJm4JIVC1alVERkaiXr16BgkqhdTUVLi6usJ/4iZcmfuu1HGIiErdlStXULt2baljkBnK+wxVKpVwcXEx6HPrfXuAevXq4e+//8by5cuxc+dOxMfHQ6PRoHLlyujYsSNGjBgBBwcHQ2aVTLMaHlJHICIyOiEEvvvuOwwYMAA+Pj4AwOJCJknvkZeyIK81Dv4lGhEjgqSOQ0RkNEIITJgwAWFhYahTpw5iY2NfejkMohcx5siLXnNeVq5cqb2nxYscP34cK1eu1OclTIqMdzciIgsmhMD48eMRFhYGAPj0009ZXMik6VVeBg8enO8mjEVZunQphgwZos9LmBSebERElkoIgbFjx2LBggUAgF9++QUjR46UOBXRi+k950UXlnJEileSJCJLJITAp59+ikWLFgEAlixZguHDh0uciujljFpekpKSLGLSLi9SR0SWaPbs2Vi0aBFkMhl+/fVXDBs2TOpIRDrRubz873//y/fz/fv3CyzLk5ubiwsXLmDv3r1o0KBByRKaAHYXIrJEgwYNwsqVKzFp0iQMHTpU6jhEOtO5vAQFBeU7fLJnzx7s2bPnhdsIIfDhhx/qn85EcOSFiCxRpUqVcO7cOU7OJbOjc3lp1aqVtrwcPHgQXl5eqFOnTqHr2tjYoFKlSnj33XfRqVMnwySVELsLEVkCjUaD0aNHo1WrVujduzcAsLiQWdK5vMTExGi/l8vl6NixI5YtW2aMTCaHIy9EZO40Gg0++OAD/PrrrwgPD0eLFi1QuXJlqWMR6UWvCbvR0dHw9vY2dBaTxVOlicicaTQajBw5EuHh4ZDL5Vi6dCmLC5k1vcpLYGCgoXOYNI68EJG50mg0GD58OJYtWwa5XI5Vq1ahX79+UsciKhGDnCqtVCqRmppa5HVdqlSpYoiXkUxKZrbUEYiIik2tVuP999/H8uXLIZfLsXr1avTt21fqWEQlpnd5SU5OxldffYWNGzfi4cOHRa4nk8mQm5ur78uYhIxsjdQRiIiKbd26dVi+fDkUCgXWrFmjnaRLZO70Ki9KpRLNmjXD9evXoVAoYG9vj4yMDPj4+OD+/fsQQkAmk5n9iEuequ7mf6E9Iip7+vXrh5MnT6Jly5YIDQ2VOg6Rweh1b6P58+fj2rVrGDhwIJRKJUJCQiCTyXD37l2kpaXhv//9L9zc3BAYGIgbN24YOnOp44RdIjIXarUa2dnPDnXLZDIsXLiQxYUsjl7lZdu2bShfvjz++9//wt7ePt/F6xwcHDBy5EhERkZi9erVWLJkicHCSoX3NiIic6BWqzFo0CCEhoZqCwyRJdKrvMTHx+P111/XXtwo78NdrVZr12nWrBmaN2+OpUuXGiAmERG9SG5uLgYOHIg1a9YgMjISJ0+elDoSkdHoVV4AoFy5ctrv826+mJycnG+dKlWq4PLly/q+hMngyAsRmbLc3Fy89957WLt2LaysrLBhwwa8+eabUsciMhq9youvry/u3r2r/TlvYu758+fzrRcfHw8rK6PeuLpUsLsQkanKzc3FgAEDsG7dOlhZWWHjxo3o0aOH1LGIjEqv8tKgQQNcuXJF+/Nbb70FIQSmTZuGtLQ0AMDq1atx4sQJ1KtXzzBJJcQJu0RkinJyctCvXz+sX78e1tbW2LRpE9555x2pYxEZnV7lpUOHDkhKSkJ0dDQAoHnz5mjZsiWOHDkCd3d3eHh4YNCgQZDJZJg4caJBA0uBh42IyBRdvnwZO3fuhLW1NTZv3ozu3btLHYmoVOhVXvr27YtDhw7B399fu2zLli3o0qULgGdzX9zc3BAWFoauXbsaJqmE2F2IyBQ1aNAAO3fuxJYtWyzi31oiXek1IcXJyQktW7bMt8zT0xPbtm1DRkYGlEolKlSoALlc7/nAJkUGthciMg05OTm4desWatasCQBo1aqVxImISp/B24WDgwN8fHy0xeX5uTHmitWFiExBdnY2evfujebNm+Ovv/6SOg6RZIw2NHL9+nUMGDAADRo0MNZLlBpO2CUiqWVnZyM0NBRbt25FWloa7t27J3UkIskU+7DRzZs38eDBA1SoUAHVqlUr9PEZM2ZgzZo1yM3NtYjJrjxsRERSUqlU6NWrF7Zv3w5bW1v88ccfaN++vdSxiCSj88jLiRMn0KBBA/j5+aFFixbw8/ND48aNcebMGQDPrjXw+eefo27duli5ciVyc3PRqlUrHDlyxGjhSwtHXohIKiqVCiEhIdi+fTvs7Oywbds2Fhcq83Qaebl16xbatm2Lp0+fQgihXX7u3Dm0b98e586dQ9++fXH48GEIIdCoUSPMnj0bHTp0MFrwUmUBo0dEZH5UKhXeffddREZGaotL27ZtpY5FJDmdRl7CwsKQnp6OmjVrYs2aNfjrr79w5MgRTJ48GWlpaQgKCsKhQ4fg5OSE8PBwnDlzxnKKCzjyQkTSyMnJgVKphL29PXbs2MHiQvQPnUZeDhw4AEdHR0RFRaFixYra5c2bN0e5cuUwceJEyGQy7N69G82bNzdaWKlwzgsRScHJyQk7d+7EhQsX0KxZM6njEJkMnUZebt26hWbNmuUrLnn69OkDAAgICLDI4gIAicpMqSMQURmRlZWF3377Tfuzs7MziwvRv+hUXp4+fYpKlSoV+ljecj8/P8OlMjGezrZSRyCiMiAzMxPdu3dHv379MG/ePKnjEJksnU+Vftkpz9bW1iUOY6rsbCzjSsFEZLryisu+ffvg4OCAgIAAqSMRmSx+KutAzrONiMiIMjIy0K1bN+zbtw+Ojo7YtWsXAgMDpY5FZLJ0Li8rVqyAQqEo9EsmkxX5uJWVXrdPgkqlwqRJk+Dr6wt7e3sEBARg3759Om+/fv16NG/eHI6OjnBzc0OLFi0QFRWlVxZWFyIylrzisn//fjg5OWH37t28XxHRS+hcXoQQen/pY/DgwQgLC0P//v2xcOFCKBQKdOrUCYcPH37pttOnT0ffvn1RuXJlhIWFYdasWWjYsCHu3r2rVxaOvBCRMajVanTr1g0HDhzQFpc333xT6lhEJk+nYZHo6Ghj58jn5MmTWLduHebPn48JEyYAAAYOHIj69etj4sSJOHr0aJHbHj9+HF9//TX+85//YOzYsQbJw+5CRMagUCjQvXt3nDx5Ert370aLFi2kjkRkFnQqL6V97HXTpk1QKBQYMWKEdpmdnR2GDRuGL774AgkJCahcuXKh237//ffw9vbGJ598AiEEnj59CicnpxLlsYT7MxGRaRo9ejRCQ0NRoUIFqaMQmQ2TnLAbGxsLf39/uLi45FvetGlTAMDZs2eL3PbAgQNo0qQJFi1aBE9PTzg7O8PHxweLFy/WOw+rCxEZSnp6Oj7++GMkJydrl7G4EBWPfrNpjSwxMRE+Pj4FluctK+pW8MnJyXj06BGOHDmCqKgoTJs2DVWqVEFERARGjx4Na2trjBw5ssjXValUUKlU2p9TU1MB8PYARGQYaWlp2rl7V65cwb59+ziyS6QHkxx5yczMhK1twQvD2dnZaR8vTHp6OgDg8ePHCA8Px4QJExAaGorIyEjUq1cPs2bNeuHrzpkzB66urtqvvENT/MeFiEoqLS0NHTt2xOHDh+Hq6orZs2fz3xYiPZlkebG3t883ApInKytL+3hR2wHPLpgXEhKiXS6Xy9G7d2/cuXMHt2/fLvJ1J0+eDKVSqf1KSEh4tj3/fSGiEkhNTUWHDh1w5MgRuLq6Yt++fdrD4ERUfCZ52MjHx6fQ05oTExMBAL6+voVu5+7uDjs7O7i5uUGhUOR7zMvLC8CzQ0tVqlQpdHtbW9tCR3x4uhER6SuvuBw7dgxubm7Yt28f3njjDaljEZk1kxx5adSoEa5evaqdc5LnxIkT2scLI5fL0ahRIzx8+BDZ2dn5HsubJ+Pp6VnsPKwuRKSvoUOH4tixYyhXrhwOHDjA4kJkACZZXkJCQqBWq7FkyRLtMpVKhYiICAQEBGjnoty+fRuXL1/Ot23v3r2hVquxYsUK7bKsrCysWbMG9erVK3LU5kV4XJqI9DVnzhw0aNAA+/fvx2uvvSZ1HCKLYJKHjQICAtCrVy9MnjwZSUlJqFmzJlasWIGbN29i6dKl2vUGDhyIgwcP5ruK78iRIxEeHo6PP/4YV69eRZUqVbBq1SrcunUL27dv1ysPqwsRFYcQQvtHT61atXD27FnI5Sb5tyKRWSrxf01KpRL79+/Hb7/99sIr3xbXypUr8emnn2LVqlUYM2YMcnJysGPHjpfe88Pe3h5RUVHo168fli1bhs8++wxyuRyRkZHo2LGjXlk4YZeIdJWcnIxWrVph9+7d2mUsLkSGJRN63nwoLS0NY8eOxapVq5CbmwsAGDRoEJYtWwYACA8Px1dffYWtW7ea7a3dU1NT4erqioioCxgcXE/qOERk4pKTk9G2bVv8+eefqFSpEq5du6a9xANRWZP3GapUKgtcdLak9PpzIDMzE0FBQVi2bBnKlSuHjh07FrgBY5cuXfDgwQP8/vvvhsgpqUdPC562TUT0vCdPnqBNmzb4888/Ub58eURGRrK4EBmJXuUlLCwMsbGx6Nu3L+Li4rBjx44C63h7e6Nu3bqlflNHY6jgzH+AiKhoecXlzJkz8PT0RHR0NBo2bCh1LCKLpVd5Wb9+Pby9vbF06VI4OjoWuZ6/vz/u3LmjdzhTwZONiKgojx8/RuvWrREbGwtPT09ERUWhfv36Uscismh6lZe4uDg0bdr0pUOiDg4OePTokV7BTAnLCxEVZeHChTh79iy8vLwQHR3N4kJUCvQ6VVqhUCAnJ+el6925c+eFIzNERObuq6++QnJyMj788EPUq8eJ/USlQa/y4ufnh3PnziE3NxdWVoU/RXp6Os6fP28R/zHLeKUXInpOSkoKnJ2doVAoYGVlhR9++EHqSERlil6Hjbp164bExMQX3qV51qxZUCqV6NGjh97hiIhMTVJSEt566y0MGzYMarVa6jhEZZJeIy9jx45FREQEZs6cibNnzyI0NBQA8ODBA2zZsgUbNmzAxo0bUa1aNXzwwQcGDSwFznkhIuBZcXn77bdx4cIFPH78GImJiahUqZLUsYjKHL0vUnfx4kV069YN8fHxBe79I4RA1apVERkZadaHjfIusLP20CX0fbOO1HGISEIPHjzA22+/jYsXL8LX1xfR0dHw9/eXOhaRyTLmRer0vrdRvXr18Pfff2P58uXYuXMn4uPjodFoULlyZXTs2BEjRoyAg4ODIbMSEUni/v37ePvtt3Hp0iVUrFgR0dHRqFWrltSxiMosvUdeygKOvBDR/fv3ERwcjMuXL6NSpUqIjo5GzZo1pY5FZPJM7vYAjx8/NmgIU/fvw2JEVHb89ddfiIuLQ6VKlRATE8PiQmQC9CovFStWRGhoKHbt2gWNRmPoTEREJqNt27b4448/EBMTAz8/P6njEBH0LC9CCGzatAldunRB5cqV8cUXX+Dq1auGzmYyOO5CVLbcu3cPcXFx2p87duzI4kJkQvQqL4mJiVi4cCFeffVVJCYmYu7cuahbty7efPNNLFu2DOnp6YbOSURUKu7evYugoCAEBQXlKzBEZDr0Ki/u7u4YPXo0zpw5g7Nnz2L06NHw8PDA0aNHMXz4cHh7e2Pw4ME4ePCgofNKglNeiMqGO3fuICgoCNeuXYOVlRUUCoXUkYioEAY72yg3Nxfbtm3D8uXLsXv3buTm5kImk6F69eq4fv26IV6i1OXNlF5/5DJCW9SWOg4RGVFCQgKCg4MRFxeHatWqISYmBlWrVpU6FpHZMrmzjQpjZWWFnj17Ytu2bbhz5w5Gjx4NIQRu3LhhqJeQDO9tRGTZbt++rT1MVL16dRw8eJDFhciE6X2RusKoVCps2bIFy5cvR1RUlCGfmojIKBISEhAUFIQbN26gRo0aiI6ORpUqVaSORUQvYJDycuLECURERGDDhg1QKpUQQsDNzQ19+vTB0KFDDfESkuKcFyLL5eDgABcXF/j5+SE6OhqVK1eWOhIRvYTe5SUxMRGrVq3C8uXLceXKFQghIJPJ8Pbbb2Po0KHo2bMnbG1tDZmViMjgPDw8sH//fmRlZfEmi0RmQq/y0qlTJ+zbtw8ajQZCCFSrVg2DBw/G4MGDLXK4lQMvRJblxo0b+N///odBgwYBAMqXLy9xIiIqDr3Ky+7du2Fvb4+ePXti6NChCA4ONnQuIiKjiI+PR3BwMG7fvg0rKyv0799f6khEVEx6lZeff/4Zffr0MfipTyaLQy9EFiE+Ph5BQUFISEiAv78///AiMlN6lZcRI0YYOgcRkVHFxcUhKCgId+7cQe3atREdHQ0fHx+pYxGRHgx2nRfLxqEXInN2/fp1BAYG4s6dO6hTpw5iYmJYXIjMmE4jL0OHDoVMJsPs2bNRoUKFYp3+LJPJsHTpUr0DEhGVxJMnTxAUFIS7d++ibt26iIqKgre3t9SxiKgEdLo9gFwuh0wmw6VLl+Dv7w+5XPcBG5lMBrVaXaKQUsm7tPHm41fRM6CW1HGISE/Tp0/Hxo0bERUVhQoVKkgdh6hMMObtAXQaeYmIiAAA7TBr3s9EROZg+vTp+Oyzz+Do6Ch1FCIyAJ3KS961EIr62dJxxguRebl8+TJmzJiB8PBwbWFhcSGyHAa9txERkdQuXbqEt99+G/fv30e5cuXw008/SR2JiAxMr7ONFAoFhg0b9tL1hg8fDisr8+9HMt7ciMgsXLx4EcHBwbh//z4aNmyIr7/+WupIRGQEepUXIQR0mOerXZeIyNjyisuDBw/QqFEjREVF8bL/RBbKqNd5ycjIgLW1tTFfolRw3IXItP39998ICgpCUlISGjdujP3798PDw0PqWERkJEY7ppOSkoLDhw/zQlBEZFRqtRq9evXCw4cPtcXF3d1d6lhEZEQ6l5caNWrk+3nTpk2IiYkpdN3c3Fzcv38farUaI0eOLFFAU8ApL0SmS6FQYO3atZg0aRLWrVvH4kJUBuhcXm7evKn9XiaTIT09Henp6UWub2Njg3feeQezZ88uUUAiosLk5ORoD0s3btwYe/fulTgREZUWncvLjRs3ADybgFujRg2EhIRg/vz5ha5rY2MDT09PizjTCODIC5GpOXfuHN555x2sWbMGLVq0kDoOEZUyndtF1apVtd8PGjQIb731Vr5lRESl4ezZs2jdujWePHmCadOmYe/evbycAVEZo9fQSFm7PYCM5xsRmYQzZ86gTZs2SE5ORkBAADZt2sTiQlQGGfVUaSIiQ3m+uDRr1gx79uyBq6ur1LGISAI6jbwMHToUMpkMs2fPRoUKFTB06FCdX0Amk2Hp0qV6BzQJ/MOOSFJ//vkn2rRpg5SUFDRv3hy7d+82+F1qich8yIQOl8CVy+WQyWS4dOkS/P39IZfrPmAjk8mgVqtLFFIqebfz3n76Orq87id1HKIya8CAAdrJubt374azs7PUkYjoJfI+Q5VKpcH/2NBp5CVvjkveBefK3pwXIpLS0qVLUaVKFUyePJnFhYh0G3kpq/Ja447T19GZIy9EperGjRuoVq0aJ+QSmSljjrxwwq4O+I8nUek6fvw4GjVqhHHjxvHmrkRUgMHLy40bN/DHH3/g7Nmzhn5qIioDjh07hnbt2iE1NRWxsbFQqVRSRyIiE6NXedm2bRt69uyJkydP5ls+f/58+Pv7o2fPnnj99deLdVaSKeO4C1HpOHr0KNq3b4+0tDQEBQUhMjISdnZ2UsciIhOjV3lZuXIldu/ejbp162qXXb58GZ9//jmEEHj11Vfh4OCAFStWYPv27QYLS0SW68iRI9riEhwcjMjISDg6Okodi4hMkF7lJTY2Fq+++mq+Wf9r1qwBAPz00084c+YMTp06BYVCgSVLlhgmqYQ45YXIuA4fPoz27dsjPT0db7/9Nnbs2AEHBwepYxGRidKrvDx69AgVK1bMtywmJgb29vYYPHgwAKBOnTp48803ceHChRKHJCLLdvv2bWRmZqJNmzbYvn07iwsRvZBe9zbKysqCQqHQ/qxWq3HmzBk0a9YMNjY22uW+vr44fvx4yVNKjPc2IjKufv36oXz58njrrbdgb28vdRwiMnF6jbx4eXnh2rVr2p+PHz+OzMxMtGzZMt96mZmZFnHMOiM7V+oIRBbnyJEjuHfvnvbndu3asbgQkU70Ki8tWrTAuXPnsG7dOiiVSsyePRsymQxt2rTJt96lS5fg6+trkKBScrLVa4CKiIoQHR2Ndu3aITg4GA8ePJA6DhGZGb3Ky6RJk2BlZYX+/fvD3d0du3btwmuvvYZWrVpp10lISMDly5fRpEkTg4UlIvMXFRWFzp07IyMjAzVq1OCdoYmo2PQqL6+99hp27tyJwMBA1K1bF4MHD8aOHTvyrbNhwwa4urqidevWBgkqJc55ITKMAwcOoHPnzsjMzETHjh2xdetWXseFiIqN9zZ6gbz7MuyLvYE2japJHYfIrO3fvx9du3ZFVlYWOnfujM2bN8PW1lbqWERkJLy3kcR4nReikomOjtYWly5durC4EFGJlHgm6vHjxxEdHY27d+8CACpWrIjg4GA0a9asxOGIyDL4+/ujUqVKqFu3LjZu3MjiQkQlond5uX37Nvr374+jR48CgPbOr3l3YG7ZsiVWr16NKlWqGCCmtDjwQlQyFStWxKFDh+Du7p7vWlBERPrQq7ykpKQgODgYN27cgJ2dHdq3bw8/Pz8AQHx8PHbv3o3Dhw+jdevWOH36NM8mICqDdu3aBaVSiT59+gAAvL29JU5ERJZCr/Lyn//8Bzdu3ECnTp2wZMmSAtdyuX//PoYPH46dO3fiP//5D77++muDhJUMh16IimXnzp3o0aMHcnNz4evrm+8yCkREJaXXhN2tW7fC09MTGzZsKPQidN7e3li/fj3Kly+PLVu2lDgkEZmPyMhI9OjRA9nZ2ejRoweaN28udSQisjB6lZcbN24gMDDwhTdPc3BwQGBgIG7cuKF3OFPBs42IdLN9+3ZtcQkJCcFvv/0Ga2trqWMRkYXRq7woFArk5OS8dL3c3FzI5Twbm6gs2LZtG959913k5OSgV69eWLt2LYsLERmFXs2iVq1aiImJQUpKSpHrPHnyBNHR0fD399c3m8ngwAvRi50/fx4hISHIyclB7969WVyIyKj0Ki+9evWCUqlE586dceHChQKP//XXX+jSpQtSU1PRu3fvEockItPWoEEDjBw5En369MHq1athZcWbmRKR8eh1e4DMzEztnaXlcjkaN26M6tWrA3h2qvTZs2eh0WjQqFEjHD161GzvXZJ3aeOYv24gsH41qeMQmRwhhPbaTkIIaDQaKBQKiVMRkSkwudsD2NvbIyoqCqGhoQCAP//8E5s2bcKmTZtw5swZAEDv3r2xf/9+sy0uRPRimzZtQq9evZCdnQ3g2QUqWVyIqDToPbZbrlw5rFu3DgkJCfjf//6X7/YArVq1QuXKlQ0WUnqc9UL0vI0bN6Jv375Qq9X45ZdfMHr0aKkjEVEZUuzy8uDBA9y6dQu2traoU6cOKleujP79+xsjGxGZoPXr16N///5Qq9UYOHAgPvroI6kjEVEZo/Nho6tXryIoKAi+vr5o3rw5XnvtNbi7u2PcuHE6nTZtznidF6Jn1q1bpy0ugwcPxrJly3ioiIhKnU4jLw8fPkRgYCCSkpLw/PzezMxMLFy4EI8ePcLKlSuNFpKIpLd27Vq899570Gg0GDJkCMLDw3kdJyKShE7/8oSFheHBgweoWLEiIiIicP78eRw+fBjjx4+HQqHAmjVrcPHiRWNnlQwHXqise/LkCT788ENoNBoMGzaMxYWIJKXTyMuuXbtgZ2eHqKgo1KxZU7u8RYsWcHNzw9SpU7F7927Uq1fPaEGJSDru7u7Yvn07Nm/ejAULFrC4EJGkdPoXKD4+Hs2aNctXXPIMHDgQACziHkZFkXHSC5VRqamp2u9btWqFhQsXsrgQkeR0+lcoPT0dVatWLfSxvFOiMzIyDJeKiCS3YsUK1KpVC+fPn5c6ChFRPjr/CfWy0Qc9LtRrNjjuQmVNREQEhgwZgqSkJKxdu1bqOERE+Zjs+K9KpcKkSZPg6+sLe3t7BAQEYN++fcV+nrZt20Imk2HUqFFGSElkeZYtW4Zhw4ZBCIGPPvoIc+bMkToSEVE+OpeXFStWQKFQFPolk8mKfFzfG7QNHjwYYWFh6N+/PxYuXAiFQoFOnTrh8OHDOj/Hli1bcOzYMb1e/3mc8kJlRXh4uLa4jBo1CosXL+acLyIyOTqXFyGE3l/FdfLkSaxbtw5z5szB/PnzMWLECERFRaFq1aqYOHGiTs+RlZWF8ePHY9KkScV+faKy6Ndff8Xw4cMBAKNHj8aiRYtYXIjIJOk0LBIdHW3sHPls2rQJCoUCI0aM0C6zs7PDsGHD8MUXXyAhIeGl906aN28eNBoNJkyYgK+++qpEeWSc9UIWTq1WY/Xq1QCATz75BAsWLGBxISKTpVN5CQwMNHaOfGJjY+Hv71/gFtpNmzYFAJw9e/aF5eX27dv49ttvsWzZMtjb2xs1K5ElUCgU2LFjB1auXImPPvqIxYWITJpJTthNTEyEj49PgeV5y+7du/fC7cePH4/GjRujT58+xXpdlUqF1NTUfF8A57yQ5Tpz5oz2e2dnZ3z88ccsLkRk8kyyvGRmZsLW1rbAcjs7O+3jRYmOjsbmzZvx/fffF/t158yZA1dXV+3Xyw5NEZmzn376Ca+//jq++eYbqaMQERWLSZYXe3t7qFSqAsuzsrK0jxcmNzcXY8aMwXvvvYcmTZoU+3UnT54MpVKp/UpISCj2cxCZg8WLF+Pjjz8GAKSkpFj0dZqIyPLodx6zkfn4+ODu3bsFlicmJgIAfH19C91u5cqVuHLlCn755RfcvHkz32NpaWm4efMmvLy84ODgUOj2tra2hY74EFmSRYsW4ZNPPgEATJw4Ed9++y0PFRGRWTHJkZdGjRrh6tWr+e6rAgAnTpzQPl6Y27dvIycnBy1btkT16tW1X8CzYlO9enXs3bu32Hn47zpZiu+//15bXD7//HMWFyIySyY58hISEoLvvvsOS5YswYQJEwA8m0wbERGBgIAA7VyU27dvIyMjA3Xq1AEA9OnTp9Bi06NHD3Tq1AnDhw9HQEBAqb0PIlOyYMECjBs3DsCzQ6TffPMNiwsRmSWTLC8BAQHo1asXJk+ejKSkJNSsWRMrVqzAzZs3sXTpUu16AwcOxMGDB7XH6+vUqaMtMv9WvXp1vPPOO6URn8gk5V3t+ssvv8TMmTNZXIjIbJlkeQGeHeaZOnUqVq1aheTkZDRs2BA7duxAq1atSj0LL1JHlmD06NF444030KxZMxYXIjJrMlHC0wyUSiVOnTqFhw8fomrVqmjRooWhskkuNTUVrq6uOHE5AU1rV5I6DlGxrVixAl27doW7u7vUUYiojMn7DFUqlQUuOltSek/YTUtLw/vvvw8vLy+0b98eAwYMQHh4uPbx8PBw+Pr6aifZmjP+kUrmaO7cuRg8eDDatm37wmsjERGZG73KS2ZmJoKCgrBs2TKUK1cOHTt2LHCdiC5duuDBgwf4/fffDZGTiIrh22+/xeeffw4A6N69O2+TQUQWRa/yEhYWhtjYWPTt2xdxcXHYsWNHgXW8vb1Rt27dUr+pozFw5IXMyezZszF58mQAwNdff13iG5MSEZkavcrL+vXr4e3tjaVLl8LR0bHI9fz9/XHnzh29wxFR8cyaNQtffvml9vupU6dKnIiIyPD0Ki9xcXFo2rSp9l5DRXFwcMCjR4/0CmZKeLYRmYPvv/9eW1Zmz56tLTFERJZGr/KiUCiQk5Pz0vXu3LnzwpEZIjKczp07w9fXF3PmzNEeNiIiskR6XefFz88P586dQ25urvbCV/+Wnp6O8+fPo169eiUKaAo454XMQa1atXDhwgW4ublJHYWIyKj0Gnnp1q0bEhMTMWvWrCLXmTVrFpRKJXr06KF3OCIqmhACX3/9NXbt2qVdxuJCRGWBXiMvY8eORUREBGbOnImzZ88iNDQUAPDgwQNs2bIFGzZswMaNG1GtWjV88MEHBg0sBQ68kKkRQuCrr77CrFmzYGtriytXrqBq1apSxyIiKhV6X2H34sWL6NatG+Lj4wtcalwIgapVqyIyMtKsDxvlXR3wzLU7aFyzotRxiAA8++9rypQpmD17NoBnN1z89NNPpQ1FRPQvxrzCrt73NqpXrx7+/vtvLF++HDt37kR8fDw0Gg0qV66Mjh07YsSIEXBwcDBkVulw6IVMhBACX375JebMmQPg2RlGn3zyicSpiIhKV4nvbWTJtCMv1++gsR9HXkhaQghMnjwZc+fOBQAsWrQIo0ePljgVEVHhTHLkpWzh0AtJb8OGDdri8sMPP2DUqFESJyIikgbLC5GZCAkJwYABA9CsWTN8/PHHUschIpKMXuWlRo0aOq8rk8kQFxenz8uYDF7nhaQihIBGo4FCoYBCocDKlSsLTJAnIipr9CovN2/efOk6MpkMQgiL+IdWo+G0ICp9QgiMHTsWycnJWLZsGRQKhUX890REVFJ6lZcbN24Uulyj0eDWrVvYsWMHfvjhB0yePBlDhgwpUUBTYK3Q61p+RHoTQuDTTz/FokWLAACDBw9GcHCwxKmIiEyDXuXlRRfDql69OoKCghAQEIC+ffsiMDCQF88iKgYhBMaMGYPFixcDAJYsWcLiQkT0HKMNKfTq1Qt169bVXo+CiF5OCIHRo0dj8eLFkMlkCA8Px/Dhw6WORURkUox6PKRu3bo4deqUMV+CyGJoNBp8/PHH+PHHH7XFZdiwYVLHIiIyOUY9Vfru3bvIzs425ksQWYy///4b4eHhkMlkWLp0qUXMFyMiMgajlZfVq1fj2LFjeP311431EkQWpWHDhtiyZQseP36MQYMGSR2HiMhk6VVehg4dWuRjaWlpuHz5Mi5evAiZTGYR913h2alkLBqNBklJSfD29gYAdOnSReJERESmT6/ysnz58peu4+LighkzZmDAgAH6vASRxdNoNBg5ciR2796NmJgY+Pn5SR2JiMgs6FVeIiIiinzMxsYGFStWRNOmTWFnZ6d3MCJLptFoMHz4cCxbtgxyuRyxsbEsL0REOtKrvPB4PJH+1Go13n//fSxfvhxyuRyrV69GSEiI1LGIiMyGXqdKDx06FBMnTjR0FiKLp1arMWzYMG1xWbNmDfr27St1LCIis6JXeVm9enWRtwggosKp1WoMHToUK1asgEKhwNq1a9GnTx+pYxERmR29Dht5e3uXqRvEyVB23isZT3p6Os6fPw+FQoHffvsNvXr1kjoSEZFZ0qu8tG3bFrt370ZOTg6sra0NnYnIIrm6umL//v04efIkOnbsKHUcIiKzpddho+nTp0OlUmH48OFIS0szdCYii5Gbm4u9e/dqf/bw8GBxISIqIb1Ple7QoQNWrlyJyMhItGnTBtWqVYO9vX2BdWUyGaZOnVrioETmJjc3FwMHDsRvv/2G//73v/jggw+kjkREZBFkQgjxspXefvttdOjQQXuGkVwuh0wmw4s2zXtcJpNBrVYbLnEpSk1NhaurK/6+kYhXqnlLHYfMSG5uLgYMGID169fDysoKGzduxDvvvCN1LCKiUpP3GapUKuHi4mLQ59Zp5CUmJgbVqlXT/vzVV1+VqQm7RMWRm5uL/v37Y8OGDbC2tsbGjRvRvXt3qWMREVkMvQ4bTZ8+3cAxiCxDTk4O+vfvj40bN8La2hqbN29G165dpY5FRGRRjHZXaUvCMSbShVqtRt++fbF582bY2Nhg8+bNvNEiEZER6HW2EREVpFAoUL9+fdjY2GDLli0sLkRERsLyQmRA06dPx99//43OnTtLHYWIyGLpXF7yLmle3C8rKx6ZIsuVnZ2N6dOn4+nTp9pltWrVkjAREZHl07lZ6HBGNVGZkp2djV69emHbtm04ceIEdu7cybPwiIhKgc7lpUOHDpg0aZIxsxCZDZVKhV69emH79u2ws7PD2LFjWVyIiEqJzuXF29sbgYGBxsxCZBZUKhXeffddREZGws7ODtu2bUPbtm2ljkVEVGZwQgpRMWRlZeHdd9/Fzp07YW9vj+3bt6N169ZSxyIiKlNYXoiK4f3339cWlx07duDtt9+WOhIRUZnDU6WJimHChAmoXLkyIiMjWVyIiCTCkReiYmjUqBGuXbsGW1tbqaMQEZVZOo28aDQaLFu2zNhZiExORkYGevbsiSNHjmiXsbgQEUmLIy864BmwZVNGRga6deuGAwcO4Pjx44iLi4O9vb3UsYiIyjyWF6JCZGRkoGvXroiKioKTkxM2bNjA4kJEZCJYXoj+5enTp+jatSuio6Ph5OSE3bt3o2XLllLHIiKif7C8ED3n6dOn6NKlC2JiYuDs7Izdu3ejRYsWUsciIqLnsLwQPWfu3Lna4rJnzx40b95c6khERPQvLC9Ez/nyyy9x/fp1jBkzBs2aNZM6DhERFYLlRQe84Z5ly8zMhJ2dHWQyGWxtbbF27VqpIxER0QvwCrtUpqWlpaFt27YYN24chBBSxyEiIh2wvFCZlZqaig4dOuDIkSNYvnw5bt++LXUkIiLSAQ8bUZmUV1yOHTsGNzc37N+/H1WrVpU6FhER6YAjL1TmKJVKtG/fHseOHUO5cuVw4MABvP7661LHIiIiHXHkhcqUvOJy4sQJuLu7Y//+/WjcuLHUsYiIqBhYXnTAc40sx+HDh3Hq1Cm4u7vjwIEDaNSokdSRiIiomFheqEzp3LkzVq9ejbp167K4EBGZKZYXsnjJycnIzMyEr68vAKBv374SJyIiopLghF2yaE+ePEGbNm0QHByMe/fuSR2HiIgMgOWFLFZecTlz5gySk5ORnJwsdSQiIjIAlheySI8fP0br1q0RGxsLLy8vREdH45VXXpE6FhERGQDnvJDFefToEdq0aYNz585pi0u9evWkjkVERAbCkRcd8L6M5uPRo0do3bo1zp07hwoVKrC4EBFZII68kEVRqVTIyMiAt7c3oqOjUadOHakjERGRgbG8kEWpWLEioqOj8fTpU9SuXVvqOEREZAQ8bERmLykpCdu2bdP+XKlSJRYXIiILxvJCZu3BgwcIDg5Gjx49sHnzZqnjEBFRKWB5IbN1//59BAcH4+LFi/Dx8UHDhg2ljkRERKWAc150IOOtGU1OXnG5fPkyKlasiJiYGNSsWVPqWEREVAo48kJmJzExUVtcKlWqxOJCRFTGcOSFzEpycjKCg4Nx5coVVK5cGdHR0fDz85M6FhERlSKWFzIrbm5uaNeuHTIzMxEdHY0aNWpIHYmIiEqZTAghpA5hqlJTU+Hq6oqrtx+gVmUvqePQP4QQePToETw9PaWOQkRERcj7DFUqlXBxcTHoc3POC5m8u3fvYvTo0cjOzgYAyGQyFhciojKMh410wHsbSSchIQHBwcGIi4uDRqPBjz/+KHUkIiKSmMmOvKhUKkyaNAm+vr6wt7dHQEAA9u3b99LttmzZgt69e6NGjRpwcHBA7dq1MX78eKSkpBg/NBnU7du3ERQUhLi4OFSvXh2TJk2SOhIREZkAky0vgwcPRlhYGPr374+FCxdCoVCgU6dOOHz48Au3GzFiBC5duoQBAwZg0aJF6NChAxYvXozmzZsjMzOzlNJTSd26dQtBQUGIj49HjRo1cPDgQVSpUkXqWEREZAJMcsLuyZMnERAQgPnz52PChAkAgKysLNSvXx9eXl44evRokdvGxMQgKCgo37KVK1di0KBB+PXXX/H+++/rnCNvstG1hAeoWYkTdktLXnG5efMm/Pz8EB0djcqVK0sdi4iIiqHMTdjdtGkTFAoFRowYoV1mZ2eHYcOG4dixY0hISChy238XFwDo0aMHAODSpUsGz0qGpdFo0LVrV21xiYmJYXEhIqJ8TLK8xMbGwt/fv0BTa9q0KQDg7NmzxXq++/fvAwDKly//wvVUKhVSU1PzfVHpksvl+PHHH9G4cWMcPHgQlSpVkjoSERGZGJMsL4mJifDx8SmwPG/ZvXv3ivV8c+fOhUKhQEhIyAvXmzNnDlxdXbVfeX/x82Qj43v+6OVbb72F06dPo2LFihImIiIiU2WS5SUzMxO2trYFltvZ2Wkf19XatWuxdOlSjB8/HrVq1XrhupMnT4ZSqdR+vejwFBlOXFwcXn/9dZw/f167TC43yV9NIiIyASZ5nRd7e3uoVKoCy7OysrSP6+LQoUMYNmwY2rdvj2+++eal69va2hZamsh44uLiEBQUhDt37mDUqFE4ePAgZLywDhERvYBJ/nnr4+ODxMTEAsvzlvn6+r70Oc6dO4du3bqhfv362LRpE6ysTLKnlWnXr19HYGAg7ty5gzp16mDDhg0sLkRE9FImWV4aNWqEq1evFpgwe+LECe3jLxIXF4cOHTrAy8sLO3fuhJOTk7Gikp6uXbuGoKAg3L17F/Xq1UNMTAy8vb2ljkVERGbAJMtLSEgI1Go1lixZol2mUqkQERGBgIAA7UTa27dv4/Lly/m2vX//Ptq1awe5XI49e/bwHjgm6N/FJSoqChUqVJA6FhERmQmTPJYSEBCAXr16YfLkyUhKSkLNmjWxYsUK3Lx5E0uXLtWuN3DgQBw8eDDfmSodOnRAfHw8Jk6ciMOHD+e7Im+FChXQtm3bUn0vVNDUqVNx7949vPLKK4iKioKXFy8ASEREujPJ8gI8uyru1KlTsWrVKiQnJ6Nhw4bYsWMHWrVq9cLtzp07BwCYN29egccCAwP1Ky+chmFQ4eHhcHZ2xuzZszkyRkRExWaStwcwFXmXNr5+5wH8KnJ0oCQePXr00osEEhGR5Shztwcgy3Lx4kXUr18fs2bNkjoKERFZAJYXMqoLFy4gODgYDx48wObNm3lnbyIiKjGWFzKav//+G8HBwUhKSkLjxo2xf/9+nS8wSEREVBSWFzKKv/76C2+//TYePnyoLS4eHh5SxyIiIgvA8qIDGU83Kpbz589ri8vrr7+O/fv3w93dXepYRERkIVheyOBOnDiBR48e4Y033sC+fftYXIiIyKBM9jovZL6GDx8OZ2dndOjQAW5ublLHISIiC8ORFzKIv/76C0+ePNH+3KdPHxYXIiIyCpYXKrEzZ84gMDAQbdq0yVdgiIiIjIHlhUrkzz//RJs2bZCcnAxbW1soFAqpIxERkYVjedGBjCcbFer06dPa4tK8eXPs2bMHrq6uUsciIiILx/JCejl16hTatm2LlJQUtGjRArt37zb4vSuIiIgKw7ONqNjyiotSqUTLli2xa9cuODs7Sx2LiIjKCJYXKjY3Nzc4OjqiQYMG2LlzJ4sLERGVKpYXKrZatWrh0KFD8PLygpOTk9RxiIiojOGcF9LJsWPHsGvXLu3PNWrUYHEhIiJJcORFB2X9ZKOjR4+iQ4cOUKlUiIqKQsuWLaWOREREZRhHXuiFjhw5gvbt2yMtLQ0tW7ZEo0aNpI5ERERlHMsLFenw4cPo0KED0tPT8fbbb2PHjh1wdHSUOhYREZVxLC9UqEOHDmmLS+vWrbF9+3Y4ODhIHYuIiIjlhQq6cOECOnbsiKdPn6Jt27YsLkREZFI4YZcKqFOnDt555x08fPgQv//+O+zt7aWOREREpMXyoosydnMjhUKBFStWICcnB3Z2dlLHISIiyoeHjQgAEBUVhZEjR0KtVgN4VmBYXIiIyBRx5IVw4MABdO3aFZmZmahXrx4++eQTqSMREREViSMvZdz+/fvRpUsXZGZmonPnzvjggw+kjkRERPRCLC9l2L59+9C1a1dkZWWha9eu2Lx5M2xtbaWORURE9EIsL2XUnj17tMWlW7du2LhxI4sLERGZBZaXMiglJQWhoaFQqVTo3r07iwsREZkVlhcdWNqJ0m5ubli7di169+6NDRs2wMbGRupIREREOpMJIYTUIUxVamoqXF1dcfPeQ1T1KS91nBLLzs5mUSEiolKR9xmqVCrh4uJi0OfmyEsZsWPHDtStWxdxcXFSRyEiIioRlpcyYPv27ejZsyfi4+Px/fffSx2HiIioRFheLNwff/yBd999Fzk5OQgNDUVYWJjUkYiIiEqE5cWC/f777wgJCUFOTg569+6NNWvWwNraWupYREREJcLyogNzvC/j1q1b0atXL+Tm5qJPnz5YvXo1rKx4NwgiIjJ/LC8WSKPRYM6cOcjNzUW/fv2watUqFhciIrIYLC8WSC6XY9euXfjqq6+wcuVKFhciIrIoLC8WJD4+Xvu9h4cHZsyYAYVCIWEiIiIiw2N5sRAbNmxA7dq18d///lfqKEREREbF8mIB1q1bh379+iE3NxcnT54EL5pMRESWjOVFBzITvrvR2rVr0b9/f6jVagwZMgTh4eGQmePpUURERDpieTFja9aswXvvvQeNRoOhQ4ciPDycc1yIiMjisbyYqdWrV2PgwIHQaDR4//338euvv0Iu5/+dRERk+fhpZ6Zu3LgBjUaD4cOH45dffmFxISKiMoMXADFTU6dORaNGjdC5c2cWFyIiKlP4qWdGIiMjkZ6erv25a9euLC5ERFTm8JNPFyZw8k5ERAS6du2Kzp07IzMzU+o4REREkmF5MQNLly7FsGHDIIRAgwYNYGdnJ3UkIiIiybC8mLjw8HC8//77EEJg9OjR+OGHH3gdFyIiKtNYXkzYkiVLMHz4cADAJ598goULF7K4EBFRmcfyYqKWLVuGkSNHAgA+/fRTLFiwgMWFiIgIPFXaZL322mtwd3fHoEGD8J///IfFhYiI6B8sLzqQojc0atQI586dQ8WKFVlciIiInsPDRibk559/xuHDh7U/V6pUicWFiIjoXzjyYiJ++OEHjBkzBk5OTjh//jyqV68udSQiIiKTxJEXE7Bw4UKMGTMGADBq1ChUq1ZN2kBEREQmjOVFYgsWLMCnn34KAPjiiy8we/ZsHioiIiJ6AZYXCYWFhWHcuHEAgC+//BKzZs1icSEiInoJlheJbN26FePHjwfw7A7RM2fOZHEhIiLSASfs6sAYlaJTp07o3Lkz3njjDUyfPt0Ir0BERGSZWF5KmRACMpkMtra2+P3332Flxf8LiIiIioOHjUrR7NmzMW7cOAghAIDFhYiISA/89Cwl33zzDaZMmQLg2SGjtm3bSpyIiIjIPHHkpRTMmjVLW1xmzZrF4kJERFQCLC9G9vXXX2Pq1KkAnh02+vLLLyVOREREZN542EgH+p7CPGPGDO2ZRN9++y0mTZpkwFRERERlE8uLkfz111/4+uuvAQBz587FxIkTJU5ERERkGVhejKRBgwZYuXIlEhMTMWHCBKnjEBERWQyWFwMSQiAtLQ0uLi4AgP79+0uciIiIyPJwwq6BCCEwZcoUNGnSBPfu3ZM6DhERkcVieTEAIYT2jtBXr17F3r17pY5ERERksXjYSAcvOtdICIHPP/8c8+bNAwAsXLgQgwcPLpVcREREZRHLSwkIITBp0iTMnz8fAPDDDz9g1KhREqciIiKybCwvehJC4LPPPsN//vMfAMDixYvx8ccfS5yKiIjI8rG86EmpVGLbtm0AgJ9++gkffvihxImIiIjKBpYXPbm5uSE6OhrR0dEYMGCA1HGIiIjKDJ5tVAxCCJw+fVr7c8WKFVlciIiISpnJlheVSoVJkybB19cX9vb2CAgIwL59+3Ta9u7duwgNDYWbmxtcXFzQvXt3xMfH651FJntWXD755BMEBATgt99+0/u5iIiIqGRMtrwMHjwYYWFh6N+/PxYuXAiFQoFOnTrh8OHDL9wuPT0dwcHBOHjwIL744gvMmDEDsbGxCAwMxOPHj/XKIoTAmDFj8MMPP0AIgYyMDL2eh4iIiAxAmKATJ04IAGL+/PnaZZmZmcLPz080b978hdvOnTtXABAnT57ULrt06ZJQKBRi8uTJxcqhVCoFADF4yFABQMhkMrF06dLivRkiIqIyKO8zVKlUGvy5TXLkZdOmTVAoFBgxYoR2mZ2dHYYNG4Zjx44hISHhhds2adIETZo00S6rU6cOWrdujQ0bNuiVZ3nEMshkMixbtgxDhw7V6zmIiIjIMEyyvMTGxsLf3197g8M8TZs2BQCcPXu20O00Gg3Onz+PN954o8BjTZs2RVxcHNLS0vTKFBERwSvnEhERmQCTPFU6MTERPj4+BZbnLSvqxodPnjyBSqV66ba1a9cudHuVSgWVSqX9WalUAgDmzv8OPXr0QGpqavHeCBERURmV95kphDD4c5tkecnMzIStrW2B5XZ2dtrHi9oOgF7bAsCcOXMwY8aMAssnfTYBkz6b8PLgRERElM/jx4/h6upq0Oc0yfJib2+fbwQkT1ZWlvbxorYDoNe2ADB58mSMGzdO+3NKSgqqVq2K27dvG3zHU+FSU1NRuXJlJCQkFDhsSMbBfV76uM9LH/d56VMqlahSpQrc3d0N/twmWV58fHxw9+7dAssTExMBAL6+voVu5+7uDltbW+16xdkWeDZiU9iojaurK3/ZS5mLiwv3eSnjPi993Oelj/u89Mnlhp9ea5ITdhs1aoSrV68WmGNy4sQJ7eOFkcvlaNCgQb6r4D6/bY0aNeDs7GzwvERERFR6TLK8hISEQK1WY8mSJdplKpUKERERCAgIQOXKlQEAt2/fxuXLlwtse+rUqXwF5sqVK4iKikKvXr1K5w0QERGR0ZjkYaOAgAD06tULkydPRlJSEmrWrIkVK1bg5s2bWLp0qXa9gQMH4uDBg/lmMn/00Uf49ddf0blzZ0yYMAHW1tYICwtDhQoVMH78+GLlsLW1xbRp0wo9lETGwX1e+rjPSx/3eenjPi99xtznMmGMc5gMICsrC1OnTsXq1auRnJyMhg0bYubMmWjfvr12naCgoALlBQDu3LmDsWPHYu/evdBoNAgKCsKCBQtQs2bN0n4bREREZGAmW16IiIiICmOSc16IiIiIisLyQkRERGaF5YWIiIjMSpksLyqVCpMmTYKvry/s7e0REBCAffv26bTt3bt3ERoaCjc3N7i4uKB79+6Ij483cmLzp+8+37JlC3r37o0aNWrAwcEBtWvXxvjx45GSkmL80GauJL/nz2vbti1kMhlGjRplhJSWpaT7fP369WjevDkcHR3h5uaGFi1aICoqyoiJzV9J9vn+/fsRHByM8uXLw83NDU2bNsWqVauMnNi8paenY9q0aejQoQPc3d0hk8mwfPlynbdPSUnBiBEj4OnpCUdHRwQHB+PMmTPFDyLKoD59+ggrKysxYcIE8csvv4jmzZsLKysrcejQoRdul5aWJmrVqiW8vLzE3LlzRVhYmKhcubKoVKmSePToUSmlN0/67nMPDw/RoEEDMXXqVPHrr7+KMWPGCBsbG1GnTh2RkZFRSunNk777/HmbN28Wjo6OAoD4+OOPjZjWMpRkn0+bNk3IZDLRq1cv8fPPP4sffvhBjBw5UqxcubIUkpsvfff5H3/8IWQymWjRooX44YcfxOLFi0WrVq0EABEWFlZK6c3PjRs3BABRpUoVERQUJACIiIgInbZVq9WiRYsWwtHRUUyfPl0sXrxY1KtXTzg7O4urV68WK0eZKy8nTpwQAMT8+fO1yzIzM4Wfn59o3rz5C7edO3euACBOnjypXXbp0iWhUCjE5MmTjZbZ3JVkn0dHRxdYtmLFCgFA/Prrr4aOajFKss+fX79atWri66+/ZnnRQUn2+bFjx4RMJuOHZjGVZJ+3bdtW+Pr6iqysLO2ynJwc4efnJxo2bGi0zOYuKytLJCYmCiGEOHXqVLHKy/r16wUAsXHjRu2ypKQk4ebmJvr27VusHGWuvHz22WdCoVAIpVKZb/ns2bMFAHH79u0it23SpIlo0qRJgeXt2rUTfn5+Bs9qKUqyzwuTmpoqAIhx48YZMqZFMcQ+nzFjhqhSpYrIyMhgedFBSfZ57969hY+Pj1Cr1UKj0Yi0tDRjx7UIJdnnAQEB4pVXXil0eUBAgMGzWqLilpdevXqJChUqCLVanW/5iBEjhIODQ74i+TJlbs5LbGws/P39C9yYq2nTpgCAs2fPFrqdRqPB+fPn8cYbbxR4rGnTpoiLi0NaWprB81oCffd5Ue7fvw8AKF++vEHyWaKS7vPbt2/j22+/xdy5c194J3b6fyXZ5wcOHECTJk2waNEieHp6wtnZGT4+Pli8eLExI5u9kuzzoKAgXLhwAVOnTsX169cRFxeHmTNn4vTp05g4caIxY5dZsbGxeO211wrcqLFp06bIyMjA1atXdX4uk7w9gDElJibCx8enwPK8Zffu3St0uydPnkClUr1029q1axswrWXQd58XZe7cuVAoFAgJCTFIPktU0n0+fvx4NG7cGH369DFKPkuk7z5PTk7Go0ePcOTIEURFRWHatGmoUqUKIiIiMHr0aFhbW2PkyJFGzW6uSvJ7PnXqVNy4cQPffPMNZs2aBQBwcHDA5s2b0b17d+MELuMSExPRqlWrAsuf//+rQYMGOj1XmSsvmZmZhd5nwc7OTvt4UdsB0Gvbsk7ffV6YtWvXYunSpZg4cSJq1aplsIyWpiT7PDo6Gps3b9bexZ10o+8+T09PBwA8fvwY69atQ+/evQE8u8lsgwYNMGvWLJaXIpTk99zW1hb+/v4ICQlBz549tTcDHjBgAPbt24dmzZoZLXdZZcjPgjJXXuzt7aFSqQosz8rK0j5e1HYA9Nq2rNN3n//boUOHMGzYMLRv3x7ffPONQTNaGn33eW5uLsaMGYP33nsPTZo0MWpGS1PSf1usra3zjSbK5XL07t0b06ZNw+3bt1GlShUjpDZvJfm3ZdSoUTh+/DjOnDmjPYwRGhqKV155BZ988gnLuxEY6rMAKIPXefHx8UFiYmKB5XnLfH19C93O3d0dtra2em1b1um7z5937tw5dOvWDfXr18emTZtgZVXmenex6LvPV65ciStXrmDkyJG4efOm9gsA0tLScPPmTWRkZBgttzkryb8tdnZ28PDwgEKhyPeYl5cXgGeHlqggffd5dnY2li5dis6dO+ebf2FtbY2OHTvi9OnTyM7ONk7oMswQnwV5ylx5adSoEa5evYrU1NR8y/NadqNGjQrdTi6Xo0GDBjh9+nSBx06cOIEaNWrA2dnZ4Hktgb77PE9cXBw6dOgALy8v7Ny5E05OTsaKajH03ee3b99GTk4OWrZsierVq2u/gGfFpnr16ti7d69Rs5urkvzb0qhRIzx8+LDAB2benA1PT0/DB7YA+u7zx48fIzc3F2q1usBjOTk50Gg0hT5GJdOoUSOcOXMGGo0m3/ITJ07AwcEB/v7+uj9ZcU6LsgTHjx8vcF2ArKwsUbNmzXynx926dUtcunQp37bffvutACBOnTqlXXb58mWhUCjEpEmTjB/eTJVknycmJooaNWoIX19fcePGjdKKbPb03eeXLl0SW7duLfAFQHTq1Els3bpV3Lt3r1Tfi7koye/5ggULBACxZMkS7bLMzExRo0YNUa9ePeOHN1P67vPc3Fzh5uYm/P39hUql0i5PS0sTlSpVEnXq1CmdN2DmXnSq9L1798SlS5dEdna2dtm6desKXOfl4cOHws3NTfTu3btYr13myosQz841t7KyEp999pn45ZdfRIsWLYSVlZU4ePCgdp3AwEDx726Xmpoq/Pz8hJeXl5g3b55YsGCBqFy5svD19RVJSUml/TbMir77/NVXXxUAxMSJE8WqVavyfe3du7e034ZZ0XefFwa8zotO9N3nGRkZ4pVXXhHW1tZiwoQJYtGiRaJJkyZCoVCInTt3lvbbMCv67vNZs2YJAKJx48ZiwYIF4rvvvhN169YVAMTq1atL+22YlR9++EHMnDlTfPjhhwKA6Nmzp5g5c6aYOXOmSElJEUIIMWjQIAEg3x+dubm5olmzZsLJyUnMmDFD/Pjjj+KVV14Rzs7O4vLly8XKUCbLS2ZmppgwYYLw9vYWtra2okmTJmL37t351inqH/WEhAQREhIiXFxchJOTk+jSpYu4du1aaUU3W/rucwBFfgUGBpbiOzA/Jfk9/zeWF92UZJ8/ePBADBo0SLi7uwtbW1sREBBQYFsqqCT7fM2aNaJp06bCzc1N2Nvbi4CAALFp06bSim62qlatWuS/y3llpbDyIoQQT548EcOGDRMeHh7CwcFBBAYG5juaoSuZEELofpCJiIiISFplbsIuERERmTeWFyIiIjIrLC9ERERkVlheiIiIyKywvBAREZFZYXkhIiIis8LyQkRERGaF5YWIiIjMCssLkY6qVasGmUz2wq/vv/9e7+efPn06ZDIZpk+fbrDMhlLYe7e1tUWVKlXQu3dvHDp0SJJceVlKe9vSUNQ+r1SpErp3744dO3ZIHZFIMlZSByAyNy1btkTNmjULfaxevXqlnKZ0Pf/eU1JScPr0aWzYsAEbN27Ed999h3Hjxkmc8JmgoCAcPHgQ0dHRCAoKkjpOiTy/z5VKJWJjY7Ft2zZs27YNY8eORVhYmEFeZ/ny5RgyZAgGDRqE5cuXG+Q5iYyF5YWomN5//30MHjxY6hiS+Pd7z8rKwsiRI7Fy5UpMnDgRXbp0Kd5t7Uvo0qVLkmxbmv69z3NzczF27FgsXrwYCxYsQN++fdGkSRPpAhJJgIeNiEhvdnZ2+PHHH+Ho6Ai1Wo0tW7aU6uvXqVMHderUKfVtpWRlZYX58+fDxcUFALB9+3aJExGVPpYXIiPYsmUL3n//fdSvXx/lypWDnZ0dqlevjqFDh+LKlSvFfr6NGzeiTZs28PDwgLW1NTw8PFCvXj0MHz4c58+fL3SbTZs2oUOHDvD09ISNjQ0qVqyIAQMG4OLFiyV9e/k4OTmhdu3aAICbN2/me2zPnj3o0qULvLy8YGNjA19fX/Tu3RunT58u9LmUSiWmTJmCBg0awNHREba2tvD19UXLli3x1VdfIScnJ9/6/563EhMTA5lMhoMHDwIAgoOD880Zef5wyL+3TUlJgb29PRQKBe7evVvk+w0JCYFMJsPChQsLPFZa+9zOzg61atUCADx48KDA4/v378fo0aPRqFEjlC9fXjtXpnfv3jh16lSB9atVq4YhQ4YAAFasWJFvnxV22K203idRkUp8b2yiMiLvNvAREREvXVehUAgHBwfxxhtviJ49e4pu3bqJGjVqCADC0dFRHDlypMA206ZNEwDEtGnT8i2fMWOGACCsrKxEq1atRN++fUWnTp1E/fr1hUwmEwsWLMi3fk5OjggNDRUAhK2trWjRooXo1auXePXVVwUAYW9vL3bt2mXQ916zZk0BQIwZM0a7bMqUKQKAkMlkomXLlqJv376iUaNGAoBQKBRi6dKl+Z7j6dOnon79+gKA8PT0FF27dhV9+vQRQUFBwtvbWwAQycnJ+bYBIJ7/Z+zSpUti0KBBokKFCgKAaN++vRg0aJD269ChQ0VuK4QQffv2FQDEnDlzCn2fjx49EjY2NsLGxkY8evRIu1yKfV6rVi0BQEydOrXAY35+fsLGxkY0btxYdOvWTfTs2VPUq1dP+3u0adOmfOuPHz9etGzZUgAQfn5++fbZ8/vCGO+TSB8sL0Q6Kk55WbdunUhPT8+3TKPRiB9//FEAEK+88orQaDT5Hi+svGRlZQl7e3vh5OQkLl++XOB1bt68KS5dupRv2RdffCEAiICAABEfH5/vsY0bNwqFQiHKlStXoAi8yIve+7lz54RcLhcAxLJly4QQQuzatUsAEHZ2dmLv3r351g8PDxcAhLW1tfj777+1y1esWCEAiI4dO4rs7Ox826jVahETEyNUKlW+5YUVECGECAwMFABEdHR0ke+psG337dsnAIg6deoUus3ChQsFAPHuu+/mW17a+/zixYtCoVAIAOLUqVMFHt+6dat48uRJocutrKyEh4eHyMjIyPdYRESEACAGDRpUZCZjvE8ifbC8EOko78OkqK/AwECdnqd58+YCgLhw4UK+5YWVl6SkJAFANGzYUKfnfvz4sbC3txd2dnbizp07ha7z0UcfCQDihx9+0Ok5hSj8gzQlJUVERkYKPz8/AUD4+vpqC1vr1q0FADFu3LhCn69Lly4CgBg+fLh22bx58wQAERYWpnMuQ5cXjUajfa9Hjx4tsE3eyNGOHTu0y0p7n+/Zs0fUqVNHABBTpkzR+fny5I0uRUZG5lv+svJirPdJpA+ebURUTEWdKv3vyZ/Xr1/H7t27cf36daSlpUGtVgP4/zkKV65ceemp1Z6enqhWrRrOnz+P8ePHY9iwYS/cJjo6GpmZmWjdujUqVqxY6DpBQUH46aefcPToUYwaNeqFr/9vQ4YM0c6NeJ6fnx82b94MR0dH5Obm4siRIwBQ5FlZw4YNw44dOxAdHa1dlnfGzLx58+Dh4YEuXbrA3d29WPlKSiaTYdCgQfj666+xfPlyNG/eXPvY2bNncfbsWfj4+KBDhw7a5VLsc4VCgdWrV6N///5Fbnfv3j1ERkbi8uXLUCqVyM3NBQBcuHABwLPfv06dOumcw9jvk6g4WF6Iiullp0qr1WqMGjUKv/zyC4QQRa6Xmpqq0+utXLkSISEhCAsLQ1hYGNzd3REQEIC2bdvivffeQ/ny5bXrxsfHAwAOHDjw0guwPXz4UKfXf97zxc3GxgZeXl5o1qwZOnToACurZ/+cPH78GFlZWQCA6tWrF/o8fn5+AJBvYmxQUBAmTZqE+fPnY9CgQZDJZKhVqxZatmyJ7t27o2vXrpDLjX+OwZAhQzBz5kysX78e33//Pezt7QEAERERAICBAwdCoVBo1y/Nff7w4UMcOnQIaWlp+PDDD1GrVi00bdq0wDYzZszAN998U2CC8/N0/f3LY+z3SVQcLC9EBrZw4UL8/PPP8Pb2RlhYGFq0aIEKFSrAzs4OANCvXz/89ttvLyw2z3vrrbdw8+ZNREZG4uDBgzh69Cj27NmDXbt2Ydq0adi6dStat24NANBoNACAmjVromXLli98Xn1OEzb2NW6+/fZbfPDBB9i+fTsOHz6MI0eOICIiAhEREWjSpAmio6Ph6OhotNcHnp15ExwcjKioKGzduhX9+vVDTk4O1q5dCwAFRkFKe58rlUr06NED0dHRCA0NxcWLF+Hg4KB9fMuWLZg+fTqcnJywePFivP322/D19YW9vT1kMhm++OILzJkzR+ffvzzGfp9ExcHyQmRgGzZsAAD88ssv6NatW4HHr127VuzntLe3R0hICEJCQgA8+8t2ypQpWLJkCYYOHYpbt24BACpXrgwAqF27tmRXSfXw8ICtrS1UKhXi4+PRsGHDAuvk/RVf2OGHatWqYfTo0Rg9ejQA4NSpUxgwYABOnTqFefPmYcaMGcZ9A3hWUKKiohAREYF+/fph+/btePToEVq0aKE9LTxPae9zV1dXrF+/HnXq1MGtW7cQFhaGKVOmaB/P+/375ptvMGLEiALb6/P7B5jG7xZRHl7nhcjAnjx5AgCoWrVqgccuXLiAs2fPlvg1PD09MW/ePADA7du3kZycDABo3bo1bGxsEBMTg6SkpBK/jj6srKzw5ptvAkCRH3LLli0D8Ow6LC/TpEkTfPTRRwCg876zsbEBAO08j+J699134erqiqioKCQkJGgPGRU230eKfe7p6aktLN999x1SUlK0j73o9y8pKQn79u0r9Dlfts9M4XeLKA/LC5GB1a1bFwDw448/aofaASAxMREDBw4s1gfqrVu3EB4eXuj8hLwrq5YrV057tdUKFSpg9OjRePr0Kbp27Yq//vqrwHYqlQrbtm3D5cuXi/W+imP8+PEAgP/+9784cOBAvseWL1+Obdu2wdraGp988ol2+datW/G///0v3z4DgJycHOzevRtA4R/IhalUqRKA/5+cWlz29vbo06cPNBoN5s6di927d8PBwQG9e/cusK5U+/yjjz5ClSpVoFQq8Z///Ee7PO/3b8mSJcjOztYuVyqVGDRoEJRKZaHPl7fPirrQnKn8bhEB4EXqiHSl63Vejh8/LmxsbAQAUbNmTREaGio6dOgg7O3txSuvvCJ69OhR6PMUdqp0bGys9pooTZo0EaGhoSI0NFQ0btxYewG48PDwfM+Tk5Mj+vXrJwAIuVwuGjduLN59913Ru3dv0bJlS+Ho6CgAFOtiYsW5xk2e5y9S9+abb4p+/fqJ1157rciL1H3yyScCgChfvrxo27at6N+/v+jWrZvw8vISAETFihVFQkJCvm1QxKnSO3bsEACEjY2N6NKlixg6dKgYNmxYvosDFrVtnuPHj+c7FX7gwIFFrivVPl+2bJkAIJydncXjx4+FEELEx8cLNzc37T579913Rbdu3YSrq6vw8fERQ4cOLfRiiCqVSvj6+goAonHjxmLgwIFi2LBhYt68eUZ9n0T6YHkh0lFxPsDPnz8vunXrJnx8fISdnZ2oVauWmDhxokhNTRWDBg3SubykpqaK77//XvTo0UPUqlVLODk5CUdHR+Hv7y8GDhwoTp8+XWSGnTt3ip49e4qKFSsKa2tr4ebmJurWrSv69Okj1q5dK54+fWqU9/68Xbt2iU6dOgkPDw9hZWUlvL29Ra9evcSJEycKrBsbGys+//xz8eabb4qKFSsKGxsb4enpKV5//XUxe/bsfFe0zfOiAvLrr7+K1157TTg4OGjXez7/y8qLEEK88sor2vVedM2YPKW9z3Nzc7VXzv3888+1y2/cuCH69+8vqlSpImxtbUXVqlXFBx98IO7fv1/klZyFEOKvv/4S3bp1E56entoLDxZ2/SJDvk8ifciEKOaUcyIiIiIJcc4LERERmRWWFyIiIjIrLC9ERERkVlheiIiIyKywvBAREZFZYXkhIiIis8LyQkRERGaF5YWIiIjMCssLERERmRWWFyIiIjIrLC9ERERkVlheiIiIyKywvBAREZFZ+T+VopDbBsTkigAAAABJRU5ErkJggg=="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 47
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "**虚线表示纯随机分类器的ROC曲线**：一个好的分类器尽可能远离该线（朝左上角）。\n",
    "\n",
    "比较分类器的一种方法是测量曲线下面积(AUC)。完美分类器的 ROC AUC **= 1**，而纯随机分类器的 ROC AUC **= 0.5**。 Scikit-Learn提供了计算 ROC AUC 的函数："
   ],
   "id": "311a2b57cef87424"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-12T09:31:32.992119Z",
     "start_time": "2025-03-12T09:31:32.972728Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "roc_auc_score(y_train_6, y_scores)"
   ],
   "id": "b3fcab7d28ff9f02",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.9898509509279209)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 48
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
